Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48e8f54c6a | ||
|
|
776c216072 | ||
|
|
c60959ea9f | ||
|
|
62044c34b1 | ||
|
|
c8d328d386 | ||
|
|
fd879e0426 | ||
|
|
27f3170022 | ||
|
|
5b1241d213 | ||
|
|
eaee1578dd | ||
|
|
9e5c5b624b | ||
|
|
640d69f5cb | ||
|
|
36876c2601 | ||
|
|
39c1594505 | ||
|
|
86e1c854fa | ||
|
|
1af89236da | ||
|
|
9741867426 | ||
|
|
b7b98f94a6 | ||
|
|
fb50d68c32 | ||
|
|
692961e019 | ||
|
|
312cc94e6d | ||
|
|
ea71e4de60 | ||
|
|
4ed9cd152c | ||
|
|
0dbb1940f0 | ||
|
|
1452a361a9 | ||
|
|
503c08f999 | ||
|
|
6bee14d6fa | ||
|
|
839abdf910 | ||
|
|
6057f2e6d9 | ||
|
|
213dbc6bc8 | ||
|
|
fc77705229 | ||
|
|
12d7db4efb | ||
|
|
d7827da5bb | ||
|
|
b2ff2edd0c | ||
|
|
13e845ae51 | ||
|
|
0c90868929 | ||
|
|
44496afc66 | ||
|
|
8becd922cf | ||
|
|
16be5b5478 | ||
|
|
1e7095995b | ||
|
|
ffb995035f | ||
|
|
ffe7185443 | ||
|
|
44036d9a4a | ||
|
|
b6a7a8c987 | ||
|
|
905592a3dc | ||
|
|
9724248a49 | ||
|
|
ed058ae193 | ||
|
|
ca5a2eaa74 | ||
|
|
35b39063ac | ||
|
|
33c347956c | ||
|
|
a259ee307f | ||
|
|
46b78406b1 | ||
|
|
872843e491 | ||
|
|
a5ba556fff | ||
|
|
44d088be1e | ||
|
|
b40985a57c | ||
|
|
f8d32f5328 | ||
|
|
ba236fe258 | ||
|
|
2b5164c956 | ||
|
|
ca2d441b6d | ||
|
|
51696bb35d | ||
|
|
bed4c03a50 | ||
|
|
e041fe130b | ||
|
|
a07a0935fe | ||
|
|
de13b6edc5 | ||
|
|
303d5d6ff6 | ||
|
|
40b30d1fb1 | ||
|
|
dae6c757b5 | ||
|
|
31ecce3a72 |
+14
-75
@@ -92,9 +92,6 @@ jobs:
|
||||
- run: sudo apt-get install npm
|
||||
- run:
|
||||
command: make buildall
|
||||
- run:
|
||||
name: check tag and version output match
|
||||
command: ./scripts/version-check.sh ./lotus
|
||||
- store_artifacts:
|
||||
path: lotus
|
||||
- store_artifacts:
|
||||
@@ -351,7 +348,7 @@ jobs:
|
||||
build-macos:
|
||||
description: build darwin lotus binary
|
||||
macos:
|
||||
xcode: "12.5.0"
|
||||
xcode: "10.0.0"
|
||||
working_directory: ~/go/src/github.com/filecoin-project/lotus
|
||||
steps:
|
||||
- prepare:
|
||||
@@ -370,6 +367,11 @@ jobs:
|
||||
name: Install Rust
|
||||
command: |
|
||||
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
||||
- run:
|
||||
name: Install jq
|
||||
command: |
|
||||
curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
|
||||
chmod +x /usr/local/bin/jq
|
||||
- run:
|
||||
name: Install hwloc
|
||||
command: |
|
||||
@@ -386,9 +388,6 @@ jobs:
|
||||
- run:
|
||||
command: make build
|
||||
no_output_timeout: 30m
|
||||
- run:
|
||||
name: check tag and version output match
|
||||
command: ./scripts/version-check.sh ./lotus
|
||||
- store_artifacts:
|
||||
path: lotus
|
||||
- store_artifacts:
|
||||
@@ -736,45 +735,6 @@ jobs:
|
||||
- packer/build:
|
||||
template: tools/packer/lotus.pkr.hcl
|
||||
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
|
||||
publish-dockerhub:
|
||||
description: publish to dockerhub
|
||||
machine:
|
||||
image: ubuntu-2004:202010-01
|
||||
parameters:
|
||||
tag:
|
||||
type: string
|
||||
default: latest
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: dockerhub login
|
||||
command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
|
||||
- run:
|
||||
name: docker build
|
||||
command: |
|
||||
docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
|
||||
if [[ ! -z $CIRCLE_SHA1 ]]; then
|
||||
docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
||||
fi
|
||||
if [[ ! -z $CIRCLE_TAG ]]; then
|
||||
docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
|
||||
fi
|
||||
- run:
|
||||
name: docker push
|
||||
command: |
|
||||
docker push filecoin/lotus:<< parameters.tag >>
|
||||
docker push filecoin/lotus-all-in-one:<< parameters.tag >>
|
||||
if [[ ! -z $CIRCLE_SHA1 ]]; then
|
||||
docker push filecoin/lotus:$CIRCLE_SHA1
|
||||
docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
|
||||
fi
|
||||
if [[ ! -z $CIRCLE_TAG ]]; then
|
||||
docker push filecoin/lotus:$CIRCLE_TAG
|
||||
docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
|
||||
fi
|
||||
|
||||
workflows:
|
||||
version: 2.1
|
||||
@@ -811,31 +771,16 @@ workflows:
|
||||
suite: itest-deadlines
|
||||
target: "./itests/deadlines_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_512mb
|
||||
suite: itest-deals_512mb
|
||||
target: "./itests/deals_512mb_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_concurrent
|
||||
suite: itest-deals_concurrent
|
||||
target: "./itests/deals_concurrent_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_max_staging_deals
|
||||
suite: itest-deals_max_staging_deals
|
||||
target: "./itests/deals_max_staging_deals_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_offline
|
||||
suite: itest-deals_offline
|
||||
target: "./itests/deals_offline_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_padding
|
||||
suite: itest-deals_padding
|
||||
target: "./itests/deals_padding_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_power
|
||||
suite: itest-deals_power
|
||||
@@ -989,7 +934,14 @@ workflows:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- build-lotus-soup
|
||||
- build-macos
|
||||
- build-macos:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /.*/
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- build-appimage:
|
||||
filters:
|
||||
branches:
|
||||
@@ -1065,16 +1017,6 @@ workflows:
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- publish-dockerhub:
|
||||
name: publish-dockerhub
|
||||
tag: stable
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /.*/
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
|
||||
nightly:
|
||||
triggers:
|
||||
@@ -1088,6 +1030,3 @@ workflows:
|
||||
- publish-snapcraft:
|
||||
name: publish-snapcraft-nightly
|
||||
channel: edge
|
||||
- publish-dockerhub:
|
||||
name: publish-dockerhub-nightly
|
||||
tag: nightly
|
||||
|
||||
+14
-60
@@ -92,9 +92,6 @@ jobs:
|
||||
- run: sudo apt-get install npm
|
||||
- run:
|
||||
command: make buildall
|
||||
- run:
|
||||
name: check tag and version output match
|
||||
command: ./scripts/version-check.sh ./lotus
|
||||
- store_artifacts:
|
||||
path: lotus
|
||||
- store_artifacts:
|
||||
@@ -351,7 +348,7 @@ jobs:
|
||||
build-macos:
|
||||
description: build darwin lotus binary
|
||||
macos:
|
||||
xcode: "12.5.0"
|
||||
xcode: "10.0.0"
|
||||
working_directory: ~/go/src/github.com/filecoin-project/lotus
|
||||
steps:
|
||||
- prepare:
|
||||
@@ -370,6 +367,11 @@ jobs:
|
||||
name: Install Rust
|
||||
command: |
|
||||
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
||||
- run:
|
||||
name: Install jq
|
||||
command: |
|
||||
curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq
|
||||
chmod +x /usr/local/bin/jq
|
||||
- run:
|
||||
name: Install hwloc
|
||||
command: |
|
||||
@@ -386,9 +388,6 @@ jobs:
|
||||
- run:
|
||||
command: make build
|
||||
no_output_timeout: 30m
|
||||
- run:
|
||||
name: check tag and version output match
|
||||
command: ./scripts/version-check.sh ./lotus
|
||||
- store_artifacts:
|
||||
path: lotus
|
||||
- store_artifacts:
|
||||
@@ -736,45 +735,6 @@ jobs:
|
||||
- packer/build:
|
||||
template: tools/packer/lotus.pkr.hcl
|
||||
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
|
||||
publish-dockerhub:
|
||||
description: publish to dockerhub
|
||||
machine:
|
||||
image: ubuntu-2004:202010-01
|
||||
parameters:
|
||||
tag:
|
||||
type: string
|
||||
default: latest
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: dockerhub login
|
||||
command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
|
||||
- run:
|
||||
name: docker build
|
||||
command: |
|
||||
docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
|
||||
if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
|
||||
docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
||||
fi
|
||||
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
|
||||
docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
|
||||
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
|
||||
fi
|
||||
- run:
|
||||
name: docker push
|
||||
command: |
|
||||
docker push filecoin/lotus:<< parameters.tag >>
|
||||
docker push filecoin/lotus-all-in-one:<< parameters.tag >>
|
||||
if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
|
||||
docker push filecoin/lotus:$CIRCLE_SHA1
|
||||
docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
|
||||
fi
|
||||
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
|
||||
docker push filecoin/lotus:$CIRCLE_TAG
|
||||
docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
|
||||
fi
|
||||
|
||||
workflows:
|
||||
version: 2.1
|
||||
@@ -844,7 +804,14 @@ workflows:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- build-lotus-soup
|
||||
- build-macos
|
||||
- build-macos:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /.*/
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- build-appimage:
|
||||
filters:
|
||||
branches:
|
||||
@@ -920,16 +887,6 @@ workflows:
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
- publish-dockerhub:
|
||||
name: publish-dockerhub
|
||||
tag: stable
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- /.*/
|
||||
tags:
|
||||
only:
|
||||
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
||||
|
||||
nightly:
|
||||
triggers:
|
||||
@@ -943,6 +900,3 @@ workflows:
|
||||
- publish-snapcraft:
|
||||
name: publish-snapcraft-nightly
|
||||
channel: edge
|
||||
- publish-dockerhub:
|
||||
name: publish-dockerhub-nightly
|
||||
tag: nightly
|
||||
|
||||
+18
-72
@@ -1,73 +1,19 @@
|
||||
comment: off
|
||||
ignore:
|
||||
# Auto generated
|
||||
- "^.*_gen.go$"
|
||||
- "^.*/mock_full.go$"
|
||||
# Old actors.
|
||||
- "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
|
||||
# Tests
|
||||
- "api/test/**"
|
||||
- "conformance/**"
|
||||
# Generators
|
||||
- "gen/**"
|
||||
- "chain/actors/agen/**"
|
||||
# Non-critical utilities
|
||||
- "api/docgen/**"
|
||||
- "api/docgen-openrpc/**"
|
||||
coverage:
|
||||
status:
|
||||
patch: off
|
||||
project:
|
||||
tools-and-tests:
|
||||
target: auto
|
||||
threshold: 1%
|
||||
informational: true
|
||||
paths:
|
||||
- "testplans"
|
||||
- "tools"
|
||||
- "system"
|
||||
- "snap"
|
||||
- "lotuspond"
|
||||
- "conformance"
|
||||
- "scripts"
|
||||
- "gen"
|
||||
- "build"
|
||||
markets:
|
||||
target: auto
|
||||
threshold: 1%
|
||||
informational: false
|
||||
paths:
|
||||
- "markets"
|
||||
- "paychmgr"
|
||||
miner:
|
||||
target: auto
|
||||
threshold: 1.5%
|
||||
informational: false
|
||||
paths:
|
||||
- "miner"
|
||||
- "storage"
|
||||
chain:
|
||||
target: auto
|
||||
threshold: 1%
|
||||
informational: false
|
||||
paths:
|
||||
- "chain"
|
||||
node:
|
||||
target: auto
|
||||
threshold: 1%
|
||||
informational: false
|
||||
paths:
|
||||
- "node"
|
||||
- "blockstore"
|
||||
- "metrics"
|
||||
- "lib"
|
||||
- "genesis"
|
||||
- "gateway"
|
||||
- "api"
|
||||
- "journal"
|
||||
cli:
|
||||
target: auto
|
||||
threshold: 1%
|
||||
informational: true
|
||||
paths:
|
||||
- "cli"
|
||||
- "cmd"
|
||||
- "**/cbor_gen.go"
|
||||
- "api/test/**/*"
|
||||
- "api/test/*"
|
||||
- "gen/**/*"
|
||||
- "gen/*"
|
||||
- "cmd/lotus-shed/*"
|
||||
- "cmd/tvx/*"
|
||||
- "cmd/lotus-pcr/*"
|
||||
- "cmd/tvx/*"
|
||||
- "cmd/lotus-chainwatch/*"
|
||||
- "cmd/lotus-health/*"
|
||||
- "cmd/lotus-fountain/*"
|
||||
- "cmd/lotus-townhall/*"
|
||||
- "cmd/lotus-stats/*"
|
||||
- "cmd/lotus-pcr/*"
|
||||
github_checks:
|
||||
annotations: false
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Create a report to help us improve
|
||||
title: "[BUG] "
|
||||
labels: hint/needs-triaging, kind/bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
(If you are not sure what the bug is, try to figure it out via a [discussion](https://github.com/filecoin-project/lotus/discussions/new) first!
|
||||
|
||||
**Version (run `lotus version`):**
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Run '...'
|
||||
2. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Logs**
|
||||
Provide daemon/miner/worker logs, and goroutines(if available) for troubleshooting.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -1,100 +0,0 @@
|
||||
name: "Bug Report"
|
||||
description: "File a bug report to help us improve"
|
||||
labels: [need/triage, kind/bug]
|
||||
body:
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to file a bug report!
|
||||
options:
|
||||
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
required: true
|
||||
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
|
||||
required: true
|
||||
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
|
||||
required: true
|
||||
- label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
|
||||
required: true
|
||||
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
|
||||
required: true
|
||||
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
|
||||
required: true
|
||||
- label: I did not make any code changes to lotus.
|
||||
required: false
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a bug for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus miner - mining and block production
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt)
|
||||
required: false
|
||||
- label: lotus miner/market - storage deal
|
||||
required: false
|
||||
- label: lotus miner/market - retrieval deal
|
||||
required: false
|
||||
- label: lotus miner/market - data transfer
|
||||
required: false
|
||||
- label: lotus client
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Lotus Version
|
||||
render: text
|
||||
description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
|
||||
placeholder: |
|
||||
e.g.
|
||||
Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0
|
||||
Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: Description
|
||||
attributes:
|
||||
label: Describe the Bug
|
||||
description: |
|
||||
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
|
||||
* What you were doding when you experienced the bug?
|
||||
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
|
||||
* What is the expected behaviour?
|
||||
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
|
||||
* For proving issues, include the output of `lotus-miner proving` info.
|
||||
* For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extraInfo
|
||||
attributes:
|
||||
label: Logging Information
|
||||
render: text
|
||||
description: |
|
||||
Please provide debug logs of the problem, remember you can get set log level control for:
|
||||
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
|
||||
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
|
||||
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: RepoSteps
|
||||
attributes:
|
||||
label: Repo Steps
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
name: Deal Making Issues
|
||||
about: Create a report for help with deal making failures.
|
||||
title: "[Deal Making Issue]"
|
||||
labels: hint/needs-triaging, area/markets
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
|
||||
Please provide all the information requested here to help us troubleshoot "deal making failed" issues.
|
||||
If the information requested is missing, we will probably have to just ask you to provide it anyway,
|
||||
before we can help debug.
|
||||
|
||||
**Basic Information**
|
||||
Including information like, Are you the client or the miner? Is this a storage deal or a retrieval deal? Is it an offline deal?
|
||||
|
||||
**Describe the problem**
|
||||
|
||||
A brief description of the problem you encountered while trying to make a deal.
|
||||
|
||||
**Version**
|
||||
|
||||
The output of `lotus --version`.
|
||||
|
||||
**Setup**
|
||||
|
||||
You miner(if applicable) and daemon setup, i.e: What hardware do you use, how much ram and etc.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Run '...'
|
||||
2. See error
|
||||
|
||||
**Deal status**
|
||||
|
||||
The output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
|
||||
|
||||
**Lotus daemon and miner logs**
|
||||
|
||||
Please go through the logs of your daemon and miner(if applicable), and include screenshots of any error/warning-like messages you find.
|
||||
|
||||
Alternatively please upload full log files and share a link here
|
||||
|
||||
** Code modifications **
|
||||
|
||||
If you have modified parts of lotus, please describe which areas were modified,
|
||||
and the scope of those modifications
|
||||
@@ -1,53 +0,0 @@
|
||||
name: Enhancement
|
||||
description: Suggest an improvement to an existing lotus feature.
|
||||
labels: [need/triage, kind/enhancement]
|
||||
body:
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to create an improvement suggestion!
|
||||
options:
|
||||
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
|
||||
required: true
|
||||
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
|
||||
required: true
|
||||
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
|
||||
required: true
|
||||
- label: I **have** a specific, actionable, and well motivated improvement to propose.
|
||||
required: true
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing an improvement request for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus miner - mining and block production
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt)
|
||||
required: false
|
||||
- label: lotus miner/market - storage deal
|
||||
required: false
|
||||
- label: lotus miner/market - retrieval deal
|
||||
required: false
|
||||
- label: lotus miner/market - data transfer
|
||||
required: false
|
||||
- label: lotus client
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- type: textarea
|
||||
id: request
|
||||
attributes:
|
||||
label: Improvement Suggestion
|
||||
description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
|
||||
placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: "[Feature Request]"
|
||||
labels: hint/needs-triaging
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -1,72 +0,0 @@
|
||||
name: Feature request
|
||||
description: Suggest an idea for lotus
|
||||
labels: [need/triage, kind/feature]
|
||||
body:
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to create a new feature request!
|
||||
options:
|
||||
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
|
||||
required: true
|
||||
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
|
||||
required: true
|
||||
- label: I **have** a specific, actionable, and well motivated feature request to propose.
|
||||
required: true
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a new feature request for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus miner - mining and block production
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt)
|
||||
required: false
|
||||
- label: lotus miner/market - storage deal
|
||||
required: false
|
||||
- label: lotus miner/market - retrieval deal
|
||||
required: false
|
||||
- label: lotus miner/market - data transfer
|
||||
required: false
|
||||
- label: lotus client
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- type: textarea
|
||||
id: request
|
||||
attributes:
|
||||
label: What is the motivation behind this feature request? Is your feature request related to a problem? Please describe.
|
||||
description: A clear and concise description of what the motivation or the problem is.
|
||||
placeholder: Ex. I'm always frustrated when [...]
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Describe the solution you'd like
|
||||
description: A clear and concise description of what you want to happen.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternates
|
||||
attributes:
|
||||
label: Describe alternatives you've considered
|
||||
description: A clear and concise description of any alternative solutions or features you've considered.
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: extra
|
||||
attributes:
|
||||
label: Additional context
|
||||
description: Add any other context, design docs or screenshots about the feature request here.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
name: Mining Issues
|
||||
about: Create a report for help with mining failures.
|
||||
title: "[Mining Issue]"
|
||||
labels: hint/needs-triaging, area/mining
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
|
||||
Please provide all the information requested here to help us troubleshoot "mining/WinningPoSt failed" issues.
|
||||
If the information requested is missing, you may be asked you to provide it.
|
||||
|
||||
**Describe the problem**
|
||||
A brief description of the problem you encountered while mining new blocks.
|
||||
|
||||
**Version**
|
||||
|
||||
The output of `lotus --version`.
|
||||
|
||||
**Setup**
|
||||
|
||||
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
|
||||
|
||||
**Lotus daemon and miner logs**
|
||||
|
||||
Please go through the logs of your daemon and miner, and include screenshots of any error/warning-like messages you find, highlighting the one has "winning post" in it.
|
||||
|
||||
Alternatively please upload full log files and share a link here
|
||||
|
||||
** Code modifications **
|
||||
|
||||
If you have modified parts of lotus, please describe which areas were modified,
|
||||
and the scope of those modifications
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
name: Proving Issues
|
||||
about: Create a report for help with proving failures.
|
||||
title: "[Proving Issue]"
|
||||
labels: area/proving, hint/needs-triaging
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
|
||||
Please provide all the information requested here to help us troubleshoot "proving/window PoSt failed" issues.
|
||||
If the information requested is missing, we will probably have to just ask you to provide it anyway,
|
||||
before we can help debug.
|
||||
|
||||
**Describe the problem**
|
||||
A brief description of the problem you encountered while proving the storage.
|
||||
|
||||
**Version**
|
||||
|
||||
The output of `lotus --version`.
|
||||
|
||||
**Setup**
|
||||
|
||||
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
|
||||
|
||||
**Proving status**
|
||||
|
||||
The output of `lotus-miner proving` info.
|
||||
|
||||
**Lotus miner logs**
|
||||
|
||||
Please go through the logs of your miner, and include screenshots of any error-like messages you find, highlighting the one has "window post" in it.
|
||||
|
||||
Alternatively please upload full log files and share a link here
|
||||
|
||||
**Lotus miner diagnostic info**
|
||||
|
||||
Please collect the following diagnostic information, and share a link here
|
||||
|
||||
* lotus-miner diagnostic info `lotus-miner info all > allinfo.txt`
|
||||
|
||||
** Code modifications **
|
||||
|
||||
If you have modified parts of lotus, please describe which areas were modified,
|
||||
and the scope of those modifications
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
name: Sealing Issues
|
||||
about: Create a report for help with sealing (commit) failures.
|
||||
title: "[Sealing Issue]"
|
||||
labels: hint/needs-triaging, area/sealing
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
|
||||
Please provide all the information requested here to help us troubleshoot "commit failed" issues.
|
||||
If the information requested is missing, we will probably have to just ask you to provide it anyway,
|
||||
before we can help debug.
|
||||
|
||||
**Describe the problem**
|
||||
A brief description of the problem you encountered while sealing a sector.
|
||||
|
||||
**Version**
|
||||
|
||||
The output of `lotus --version`.
|
||||
|
||||
**Setup**
|
||||
|
||||
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
|
||||
|
||||
**Commands**
|
||||
|
||||
Commands you ran.
|
||||
|
||||
**Sectors status**
|
||||
|
||||
The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
|
||||
|
||||
**Lotus miner logs**
|
||||
|
||||
Please go through the logs of your miner, and include screenshots of any error-like messages you find.
|
||||
|
||||
Alternatively please upload full log files and share a link here
|
||||
|
||||
**Lotus miner diagnostic info**
|
||||
|
||||
Please collect the following diagnostic information, and share a link here
|
||||
|
||||
* lotus-miner diagnostic info `lotus-miner info all > allinfo`
|
||||
|
||||
** Code modifications **
|
||||
|
||||
If you have modified parts of lotus, please describe which areas were modified,
|
||||
and the scope of those modifications
|
||||
@@ -2,7 +2,7 @@ name: Close and mark stale issue
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 12 * * *'
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
@@ -18,16 +18,10 @@ jobs:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 24 hours.'
|
||||
close-issue-message: 'This issue was closed because it is missing author input.'
|
||||
stale-pr-message: 'Thank you for submitting the PR and contributing to lotus! Lotus maintainers need more of your input before merging it, please address the suggested changes or reply to the comments or this PR will be closed in 48 hours. You are always more than welcome to reopen the PR later as well!'
|
||||
close-pr-message: 'This PR was closed because it is missing author input. Please feel free to reopen the PR when you get to it! Thank you for your interest in contributing to lotus!'
|
||||
stale-issue-label: 'kind/stale'
|
||||
stale-pr-label: 'kind/stale'
|
||||
any-of-labels: 'need/author-input '
|
||||
days-before-issue-stale: 3
|
||||
any-of-labels: 'hint/needs-author-input'
|
||||
days-before-issue-stale: 5
|
||||
days-before-issue-close: 1
|
||||
days-before-pr-stale: 5
|
||||
days-before-pr-close: 2
|
||||
remove-stale-when-updated: true
|
||||
enable-statistics: true
|
||||
|
||||
|
||||
|
||||
|
||||
-693
@@ -1,698 +1,5 @@
|
||||
# Lotus changelog
|
||||
|
||||
# v1.11.3 / 2021-09-29
|
||||
|
||||
lotus v1.11.3 is a feature release that's **highly recommended to ALL lotus users to upgrade**, including node
|
||||
operators, storage providers and clients. It includes many improvements and bug fixes that result in perf
|
||||
improvements in different area, like deal making, sealing and so on.
|
||||
|
||||
## Highlights
|
||||
|
||||
- 🌟🌟Introduce `MaxStagingDealsBytes - reject new deals if our staging deals area is full ([filecoin-project/lotus#7276](https://github.com/filecoin-project/lotus/pull/7276))
|
||||
- Set `MaxStagingDealsBytes` under the [Dealmaking] section of the markets' subsystem's `config.toml` to reject new incoming deals when the `deal-staging` directory of market subsystem's repo gets too large.
|
||||
- 🌟🌟miner: Command to list/remove expired sectors locally ([filecoin-project/lotus#7140](https://github.com/filecoin-project/lotus/pull/7140))
|
||||
- run `./lotus-miner sectors expired -h` for more details.
|
||||
- 🚀update to ffi to update-bellperson-proofs-v9-0-2 ([filecoin-project/lotus#7369](https://github.com/filecoin-project/lotus/pull/7369))
|
||||
- MinerX fellows(early testers of lotus releases) have reported faster WindowPoSt computation!
|
||||
- 🌟dealpublisher: Fully validate deals before publishing ([filecoin-project/lotus#7234](https://github.com/filecoin-project/lotus/pull/7234))
|
||||
- This excludes the expired deals before sending out a PSD message which reduces the chances of PSD message failure due to invalid deals.
|
||||
- 🌟Simple alert system; FD limit alerts ([filecoin-project/lotus#7108](https://github.com/filecoin-project/lotus/pull/7108))
|
||||
|
||||
## New Features
|
||||
|
||||
- feat(ci): include version/cli checks in tagged releases ([filecoin-project/lotus#7331](https://github.com/filecoin-project/lotus/pull/7331))
|
||||
- Show deal sizes is sealing sectors ([filecoin-project/lotus#7261](https://github.com/filecoin-project/lotus/pull/7261))
|
||||
- config for disabling NAT port mapping ([filecoin-project/lotus#7204](https://github.com/filecoin-project/lotus/pull/7204))
|
||||
- Add optional mined block list to miner info ([filecoin-project/lotus#7202](https://github.com/filecoin-project/lotus/pull/7202))
|
||||
- Shed: Create a verifreg command for when VRK isn't a multisig ([filecoin-project/lotus#7099](https://github.com/filecoin-project/lotus/pull/7099))
|
||||
|
||||
## Improvements
|
||||
|
||||
- build macOS CI ([filecoin-project/lotus#7307](https://github.com/filecoin-project/lotus/pull/7307))
|
||||
- itests: remove cid equality comparison ([filecoin-project/lotus#7292](https://github.com/filecoin-project/lotus/pull/7292))
|
||||
- Add partition info to the 'sectors status' command ([filecoin-project/lotus#7246](https://github.com/filecoin-project/lotus/pull/7246))
|
||||
- chain: Cleanup consensus logic ([filecoin-project/lotus#7255](https://github.com/filecoin-project/lotus/pull/7255))
|
||||
- builder: Handle chainstore config in ConfigFullNode ([filecoin-project/lotus#7232](https://github.com/filecoin-project/lotus/pull/7232))
|
||||
- gateway: check tipsets in ChainGetPath ([filecoin-project/lotus#7230](https://github.com/filecoin-project/lotus/pull/7230))
|
||||
- Refactor events subsystem ([filecoin-project/lotus#7000](https://github.com/filecoin-project/lotus/pull/7000))
|
||||
- test: re-enable disabled tests ([filecoin-project/lotus#7211](https://github.com/filecoin-project/lotus/pull/7211))
|
||||
- Reduce lotus-miner startup spam ([filecoin-project/lotus#7205](https://github.com/filecoin-project/lotus/pull/7205))
|
||||
- Catch deal slashed because sector was terminated ([filecoin-project/lotus#7201](https://github.com/filecoin-project/lotus/pull/7201))
|
||||
- Insert miner and network power data as gibibytes to avoid int64 overflows ([filecoin-project/lotus#7194](https://github.com/filecoin-project/lotus/pull/7194))
|
||||
- sealing: Check piece CIDs after AddPiece ([filecoin-project/lotus#7185](https://github.com/filecoin-project/lotus/pull/7185))
|
||||
- markets: OnDealExpiredOrSlashed - get deal by proposal instead of deal ID ([filecoin-project/lotus#5431](https://github.com/filecoin-project/lotus/pull/5431))
|
||||
- Incoming: improve a log message ([filecoin-project/lotus#7181](https://github.com/filecoin-project/lotus/pull/7181))
|
||||
- journal: make current log file have a fixed named (#7112) ([filecoin-project/lotus#7112](https://github.com/filecoin-project/lotus/pull/7112))
|
||||
- call string.Repeat always with positive int ([filecoin-project/lotus#7104](https://github. com/filecoin-project/lotus/pull/7104))
|
||||
- itests: support larger sector sizes; add large deal test. ([filecoin-project/lotus#7148](https://github.com/filecoin-project/lotus/pull/7148))
|
||||
- Ignore nil throttler ([filecoin-project/lotus#7169](https://github.com/filecoin-project/lotus/pull/7169))
|
||||
|
||||
## Bug Fixes
|
||||
|
||||
- fix: escape periods to match actual periods in version
|
||||
- fix bug for CommittedCapacitySectorLifetime ([filecoin-project/lotus#7337](https://github.com/filecoin-project/lotus/pull/7337))
|
||||
- fix a panic in HandleRecoverDealIDs ([filecoin-project/lotus#7336](https://github.com/filecoin-project/lotus/pull/7336))
|
||||
- fix index out of range ([filecoin-project/lotus#7273](https://github.com/filecoin-project/lotus/pull/7273))
|
||||
- fix: correctly handle null blocks when detecting an expensive fork ([filecoin-project/lotus#7210](https://github.com/filecoin-project/lotus/pull/7210))
|
||||
- fix: make lotus soup use the correct dependencies ([filecoin-project/lotus#7221](https://github.com/filecoin-project/lotus/pull/7221))
|
||||
- fix: init restore adds empty storage.json ([filecoin-project/lotus#7025](https://github.com/filecoin-project/lotus/pull/7025))
|
||||
- fix: disable broken testground integration test ([filecoin-project/lotus#7187](https://github.com/filecoin-project/lotus/pull/7187))
|
||||
- fix TestDealPublisher ([filecoin-project/lotus#7173](https://github.com/filecoin-project/lotus/pull/7173))
|
||||
- fix: make TestTimedCacheBlockstoreSimple pass reliably ([filecoin-project/lotus#7174](https://github.com/filecoin-project/lotus/pull/7174))
|
||||
- Fix throttling bug ([filecoin-project/lotus#7177](https://github.com/filecoin-project/lotus/pull/7177))
|
||||
- sealing: Fix sector state accounting with FinalizeEarly ([filecoin-project/lotus#7256](https://github.com/filecoin-project/lotus/pull/7256))
|
||||
- docker entrypoint.sh missing variable escape character ([filecoin-project/lotus#7291](https://github.com/filecoin-project/lotus/pull/7291))
|
||||
- sealing: Fix retry loop in SubmitCommitAggregate ([filecoin-project/lotus#7245](https://github.com/filecoin-project/lotus/pull/7245))
|
||||
- sectors expired: Handle precomitted and unproven sectors correctly ([filecoin-project/lotus#7236](https://github.com/filecoin-project/lotus/pull/7236))
|
||||
- stores: Fix reserved disk usage log spam ([filecoin-project/lotus#7233](https://github.com/filecoin-project/lotus/pull/7233))
|
||||
|
||||
|
||||
## Dependency Updates
|
||||
|
||||
- github.com/filecoin-project/go-fil-markets (v1.8.1 -> v1.12.0):
|
||||
- github.com/filecoin-project/go-data-transfer (v1.7.8 -> v1.10.1):
|
||||
- update to ffi to update-bellperson-proofs-v9-0-2 ([filecoin-project/lotus#7369](https://github.com/filecoin-project/lotus/pull/7369))
|
||||
- fix(deps): use go-graphsync v0.9.3 with hotfix
|
||||
- Update to unified go-graphsync v0.9.0 ([filecoin-project/lotus#7197](https://github.com/filecoin-project/lotus/pull/7197))
|
||||
|
||||
## Others
|
||||
|
||||
- v1.11.3-rc2 ([filecoin-project/lotus#7371](https://github.com/filecoin-project/lotus/pull/7371))
|
||||
- v1.11.3-rc1 ([filecoin-project/lotus#7299](https://github.com/filecoin-project/lotus/pull/7299))
|
||||
- Increase threshold from 0.5% to 1% ([filecoin-project/lotus#7262](https://github.com/filecoin-project/lotus/pull/7262))
|
||||
- ci: exclude cruft from code coverage ([filecoin-project/lotus#7189](https://github.com/filecoin-project/lotus/pull/7189))
|
||||
- Bump version to v1.11.3-dev ([filecoin-project/lotus#7180](https://github.com/filecoin-project/lotus/pull/7180))
|
||||
- test: disable flaky TestBatchDealInput ([filecoin-project/lotus#7176](https://github.com/filecoin-project/lotus/pull/7176))
|
||||
- Turn off patch ([filecoin-project/lotus#7172](https://github.com/filecoin-project/lotus/pull/7172))
|
||||
- test: disable flaky TestSimultaneousTransferLimit ([filecoin-project/lotus#7153](https://github.com/filecoin-project/lotus/pull/7153))
|
||||
|
||||
|
||||
## Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @magik6k | 39 | +3311/-1825 | 179 |
|
||||
| @Stebalien | 23 | +1935/-1417 | 84 |
|
||||
| @dirkmc | 12 | +921/-732 | 111 |
|
||||
| @dirkmc | 12 | +663/-790 | 30 |
|
||||
| @hannahhoward | 3 | +482/-275 | 46 |
|
||||
| @travisperson | 1 | +317/-65 | 5 |
|
||||
| @jennijuju | 11 | +223/-126 | 24 |
|
||||
| @hannahhoward | 7 | +257/-55 | 16 |
|
||||
| @nonsense| 9 | +258/-37 | 19 |
|
||||
| @raulk | 4 | +127/-36 | 13 |
|
||||
| @raulk | 1 | +43/-60 | 15 |
|
||||
| @arajasek | 4 | +74/-8 | 10 |
|
||||
| @Frank | 2 | +68/-8 | 3 |
|
||||
| @placer14| 2 | +52/-1 | 4 |
|
||||
| @ldoublewood | 2 | +15/-13 | 3 |
|
||||
| @lanzafame | 1 | +16/-2 | 1 |
|
||||
| @aarshkshah1992 | 2 | +11/-6 | 2 |
|
||||
| @ZenGround0 | 2 | +7/-6 | 2 |
|
||||
| @ognots | 1 | +0/-10 | 2 |
|
||||
| @KAYUII | 2 | +4/-4 | 2 |
|
||||
| @lanzafame | 1 | +6/-0 | 1 |
|
||||
| @jacobheun | 1 | +3/-3 | 1 |
|
||||
| @frank | 1 | +4/-0 | 1 |
|
||||
|
||||
|
||||
# v1.11.2 / 2021-09-06
|
||||
|
||||
lotus v1.11.2 is a feature release that's **highly recommended ALL lotus users to upgrade**, including node operators,
|
||||
storage providers and clients.
|
||||
|
||||
## Highlights
|
||||
- 🌟🌟🌟 Introduce Dagstore and CARv2 for deal-making (#6671) ([filecoin-project/lotus#6671](https://github.com/filecoin-project/lotus/pull/6671))
|
||||
- **[lotus miner markets' Dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview)** is a
|
||||
component of the `markets` subsystem in lotus-miner. It is a sharded store to hold large IPLD graphs efficiently,
|
||||
packaged as location-transparent attachable CAR files and it replaces the former Badger staging blockstore. It
|
||||
is designed to provide high efficiency and throughput, and minimize resource utilization during deal-making operations.
|
||||
The dagstore also leverages the indexing features of [CARv2](https://github.com/ipld/ipld/blob/master/specs/transport/car/carv2/index.md) to enable plan CAR files to act as read and write
|
||||
blockstores, which are served as the direct medium for data exchanges in markets for both storage and retrieval
|
||||
deal making without requiring intermediate buffers.
|
||||
- In the future, lotus will leverage and interact with Dagstore a lot for new features and improvements for deal
|
||||
making, therefore, it's highly recommended to lotus users to go through [Lotus Miner: About the markets dagstore](https://docs.filecoin.io/mine/lotus/dagstore/#conceptual-overview) thoroughly to learn more about Dagstore's
|
||||
conceptual overview, terminology, directory structure, configuration and so on.
|
||||
- **Note**:
|
||||
- When you first start your lotus-miner or market subsystem with this release, a one-time/first-time **dagstore migration** will be triggered which replaces the former Badger staging blockstore with dagstore. We highly
|
||||
recommend storage providers to read this [section](https://docs.filecoin.io/mine/lotus/dagstore/#first-time-migration) to learn more about
|
||||
what the process does, what to expect and how monitor it.
|
||||
- It is highly recommended to **wait all ongoing data transfer to finish or cancel inbound storage deals that
|
||||
are still transferring**, using the `lotus-miner data-transfers cancel` command before upgrade your market nodes. Reason being that the new dagstore changes attributes in the internal deal state objects, and the paths to the staging CARs where the deal data was being placed will be lost.
|
||||
- ‼️Having your dags initialized will become important in the near feature for you to provide a better storage
|
||||
and retrieval service. We'd suggest you to start [forced bulk initialization] soon if possible as this process
|
||||
places relatively high IP workload on your storage system and is better to be carried out gradually and over a
|
||||
longer timeframe. Read how to do properly perform a force bulk initialization [here](https://docs.filecoin.io/mine/lotus/dagstore/#forcing-bulk-initialization).
|
||||
- ⏮ Rollback Alert(from v1.11.2-rcX to any version lower): If a storages deal is initiated with M1/v1.11.2(-rcX)
|
||||
release, it needs to get to the `StorageDealAwaitingPrecommit` state before you can do a version rollback or the markets process may panic.
|
||||
- 💙 **Special thanks to [MinerX fellows for testing and providing valuable feedbacks](https://github.com/filecoin-project/lotus/discussions/6852) for Dagstore in the past month!**
|
||||
- 🌟🌟 rpcenc: Support reader redirect ([filecoin-project/lotus#6952](https://github.com/filecoin-project/lotus/pull/6952))
|
||||
- This allows market processes to send piece bytes directly to workers involved on `AddPiece`.
|
||||
- Extending sectors: more practical and flexible tools ([filecoin-project/lotus#6097](https://github.com/filecoin-project/lotus/pull/6097))
|
||||
- `lotus-miner sectors check-expire` to inspect expiring sectors.
|
||||
- `lotus-miner sectors renew` for renewing expiring sectors, see the command help menu for customizable option
|
||||
like `extension`, `new-expiration` and so on.
|
||||
- ‼️ MpoolReplaceCmd ( lotus mpool replace`) now takes FIL for fee-limit ([filecoin-project/lotus#6927](https://github.com/filecoin-project/lotus/pull/6927))
|
||||
- Drop townhall/chainwatch ([filecoin-project/lotus#6912](https://github.com/filecoin-project/lotus/pull/6912))
|
||||
- ChainWatch is no longer supported by lotus.
|
||||
- Configurable CC Sector Expiration ([filecoin-project/lotus#6803](https://github.com/filecoin-project/lotus/pull/6803))
|
||||
- Set `CommittedCapacitySectorLifetime` in lotus-miner/config.toml to specify the default expiration for a new CC
|
||||
sector, value must be between 180-540 days inclusive.
|
||||
|
||||
## New Features
|
||||
- api/command for encoding actor params ([filecoin-project/lotus#7150](https://github.com/filecoin-project/lotus/pull/7150))
|
||||
- shed: Support raw encoding in cid id ([filecoin-project/lotus#7149](https://github.com/filecoin-project/lotus/pull/7149))
|
||||
- feat(miner deals): create subdir to miner repo for staged deals ([filecoin-project/lotus#6853](https://github.com/filecoin-project/lotus/pull/6853))
|
||||
- Support --actor in miner actor control list ([filecoin-project/lotus#7027](https://github.com/filecoin-project/lotus/pull/7027))
|
||||
- Shed: Include network name in genesis-verify ([filecoin-project/lotus#7019](https://github.com/filecoin-project/lotus/pull/7019))
|
||||
- feat: add ChainGetTipSetAfterHeight ([filecoin-project/lotus#6990](https://github.com/filecoin-project/lotus/pull/6990))
|
||||
- lotus-shed splitstore clear command ([filecoin-project/lotus#6967](https://github.com/filecoin-project/lotus/pull/6967))
|
||||
|
||||
## Improvements
|
||||
- improve get api error messages ([filecoin-project/lotus#7088](https://github.com/filecoin-project/lotus/pull/7088))
|
||||
- Strict major minor version checking on v0 and v1 apis ([filecoin-project/lotus#7038](https://github.com/filecoin-project/lotus/pull/7038))
|
||||
- make lotus-miner net commands hit markets subsystem. ([filecoin-project/lotus#7042](https://github.com/filecoin-project/lotus/pull/7042))
|
||||
- Test with latest actors version ([filecoin-project/lotus#6998](https://github.com/filecoin-project/lotus/pull/6998))
|
||||
- Reduce splitstore memory usage during chain walks ([filecoin-project/lotus#6949](https://github.com/filecoin-project/lotus/pull/6949))
|
||||
- Remove forgotten non-functioning config from the pre-mainnet days ([filecoin-project/lotus#6970](https://github.com/filecoin-project/lotus/pull/6970))
|
||||
- add explicit error msg if repo dir does not exist ([filecoin-project/lotus#6909](https://github.com/filecoin-project/lotus/pull/6909))
|
||||
- Test/pledge batching msg prop ([filecoin-project/lotus#6537](https://github.com/filecoin-project/lotus/pull/6537))
|
||||
- reasonable max value for initial sector expiration ([filecoin-project/lotus#6099](https://github.com/filecoin-project/lotus/pull/6099))
|
||||
- support MARKETS_API_INFO env var, and markets-repo, markets-api-url CLI flags. ([filecoin-project/lotus#6936](https://github.com/filecoin-project/lotus/pull/6936))
|
||||
- Improve formatting of workers CLI ([filecoin-project/lotus#6942](https://github.com/filecoin-project/lotus/pull/6942))
|
||||
- make: set default GOCC earlier ([filecoin-project/lotus#6932](https://github.com/filecoin-project/lotus/pull/6932))
|
||||
- Moving GC Followup ([filecoin-project/lotus#6905](https://github.com/filecoin-project/lotus/pull/6905))
|
||||
- Log more call context during errors ([filecoin-project/lotus#6918](https://github.com/filecoin-project/lotus/pull/6918))
|
||||
- polish(errors): better state tree errors ([filecoin-project/lotus#6923](https://github.com/filecoin-project/lotus/pull/6923))
|
||||
- adding an RuntimeSubsystems API to storage miner; fix `lotus-miner info` ([filecoin-project/lotus#6906](https://github.com/filecoin-project/lotus/pull/6906))
|
||||
- Reduce entropy in the chain package ([filecoin-project/lotus#6889](https://github.com/filecoin-project/lotus/pull/6889))
|
||||
- make: Allow setting Go compiler with GOCC ([filecoin-project/lotus#6911](https://github.com/filecoin-project/lotus/pull/6911))
|
||||
|
||||
## Bug Fixes
|
||||
- sealing: Fix RecoverDealIDs loop with changed PieceCID ([filecoin-project/lotus#7117](https://github.com/filecoin-project/lotus/pull/7117))
|
||||
- Fix error handling in SectorAddPieceToAny api impl ([filecoin-project/lotus#7135](https://github.com/filecoin-project/lotus/pull/7135))
|
||||
- add rice box to required binaries ([filecoin-project/lotus#7125](https://github.com/filecoin-project/lotus/pull/7125))
|
||||
- fix(miner): always create miner deal staging directory (#7098) ([filecoin-project/lotus#7098](https://github.com/filecoin-project/lotus/pull/7098))
|
||||
- fix build after merging #6097. (#7096) ([filecoin-project/lotus#7096](https://github.com/filecoin-project/lotus/pull/7096))
|
||||
- fix: don't check for t_aux when proving ([filecoin-project/lotus#7011](https://github.com/filecoin-project/lotus/pull/7011))
|
||||
- fix: vet actors shims ([filecoin-project/lotus#6999](https://github.com/filecoin-project/lotus/pull/6999))
|
||||
- fix: more logging in maybeStartBatch error ([filecoin-project/lotus#6996](https://github.com/filecoin-project/lotus/pull/6996))
|
||||
- fix flaky TestDealPublisher and re-enable ([filecoin-project/lotus#6991](https://github.com/filecoin-project/lotus/pull/6991))
|
||||
- fix skipCount ([filecoin-project/lotus#6940](https://github.com/filecoin-project/lotus/pull/6940))
|
||||
- fix bug in MpoolPending message exclusion ([filecoin-project/lotus#6945](https://github.com/filecoin-project/lotus/pull/6945))
|
||||
- PreCommitPolicy: Don't try to align expirations on proving period boundaries ([filecoin-project/lotus#7018](https://github.com/filecoin-project/lotus/pull/7018))
|
||||
- make: fix version check when using gotip ([filecoin-project/lotus#6916](https://github.com/filecoin-project/lotus/pull/6916))
|
||||
- fix ticket check ([filecoin-project/lotus#6882](https://github.com/filecoin-project/lotus/pull/6882))
|
||||
|
||||
## Dependency Updates
|
||||
- github.com/filecoin-project/go-data-transfer (v1.7.2 -> v1.7.8):
|
||||
- github.com/filecoin-project/go-fil-markets (v1.6.2 -> v1.8.1):
|
||||
- update go-libp2p-pubsub to v0.5.4 ([filecoin-project/lotus#6958](https://github.com/filecoin-project/lotus/pull/6958))
|
||||
- integrate the proof patch: tag proofs-v9-revert-deps-hotfix
|
||||
- Update markets, dt and graphsync ([filecoin-project/lotus#7160](https://github.com/filecoin-project/lotus/pull/7160))
|
||||
- Remove replace directive for multihash dep (#7113) ([filecoin-project/lotus#7113](https://github.com/filecoin-project/lotus/pull/7113))
|
||||
- upgrade upstream dependencies. ([filecoin-project/lotus#7115](https://github.com/filecoin-project/lotus/pull/7115))
|
||||
- Update to latest FFI ([filecoin-project/lotus#7110](https://github.com/filecoin-project/lotus/pull/7110))
|
||||
- Update state machine deps for logging ([filecoin-project/lotus#6941](https://github.com/filecoin-project/lotus/pull/6941))
|
||||
- Update deps for more logging in data transfer and markets ([filecoin-project/lotus#6937](https://github.com/filecoin-project/lotus/pull/6937))
|
||||
- Update to branches with improved logging ([filecoin-project/lotus#6919](https://github.com/filecoin-project/lotus/pull/6919))
|
||||
- update go-libp2p-pubsub to v0.5.3 ([filecoin-project/lotus#6907](https://github.com/filecoin-project/lotus/pull/6907))
|
||||
|
||||
## Others
|
||||
- Fix nits and see if codecov works now with auto ([filecoin-project/lotus#7151](https://github.com/filecoin-project/lotus/pull/7151))
|
||||
- Codecov Projects ([filecoin-project/lotus#7147](https://github.com/filecoin-project/lotus/pull/7147))
|
||||
- remove m1 templates and make area selection multi-optionable ([filecoin-project/lotus#7121](https://github.com/filecoin-project/lotus/pull/7121))
|
||||
- release -> master ([filecoin-project/lotus#7105](https://github.com/filecoin-project/lotus/pull/7105))
|
||||
- Lotus release process - how we make releases ([filecoin-project/lotus#6944](https://github.com/filecoin-project/lotus/pull/6944))
|
||||
- codecov: fix mock name ([filecoin-project/lotus#7039](https://github.com/filecoin-project/lotus/pull/7039))
|
||||
- codecov: fix regexes ([filecoin-project/lotus#7037](https://github.com/filecoin-project/lotus/pull/7037))
|
||||
- chore: disable flaky test ([filecoin-project/lotus#6957](https://github.com/filecoin-project/lotus/pull/6957))
|
||||
- set buildtype in nerpa and butterfly ([filecoin-project/lotus#6085](https://github.com/filecoin-project/lotus/pull/6085))
|
||||
- release v1.11.1 backport -> master ([filecoin-project/lotus#6929](https://github.com/filecoin-project/lotus/pull/6929))
|
||||
- chore: fixup issue templates ([filecoin-project/lotus#6899](https://github.com/filecoin-project/lotus/pull/6899))
|
||||
- bump master version to v1.11.2-dev ([filecoin-project/lotus#6903](https://github.com/filecoin-project/lotus/pull/6903))
|
||||
- releases -> master for v1.11.0 ([filecoin-project/lotus#6894](https://github.com/filecoin-project/lotus/pull/6894))
|
||||
|
||||
|
||||
Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @magik6k | 23 | +5040/-8389 | 114 |
|
||||
| @aarshkshah1992 | 11 | +4859/-1078 | 101 |
|
||||
| @raulk | 5 | +4170/-1662 | 104 |
|
||||
| @vyzo | 30 | +1092/-702 | 49 |
|
||||
| @nonsense | 6 | +630/-472 | 19 |
|
||||
| @ZenGround0 | 31 | +556/-274 | 74 |
|
||||
| @He Weidong | 16 | +680/-128 | 16 |
|
||||
| @raulk | 16 | +444/-277 | 49 |
|
||||
| @Stebalien | 11 | +403/-259 | 48 |
|
||||
| @jennijuju| 17 | +276/-281 | 42 |
|
||||
| @dirkmc | 5 | +204/-138 | 20 |
|
||||
| @placer14 | 7 | +178/-77 | 17 |
|
||||
| @BlocksOnAChain | 1 | +138/-0 | 1 |
|
||||
| @Frrist | 1 | +63/-56 | 2 |
|
||||
| @arajasek | 7 | +74/-42 | 13 |
|
||||
| @frrist | 2 | +67/-6 | 6 |
|
||||
| @hannahhoward | 2 | +13/-11 | 3 |
|
||||
| @coryschwartz | 1 | +16/-6 | 3 |
|
||||
| @whyrusleeping | 1 | +7/-7 | 1 |
|
||||
| @hunjixin | 1 | +8/-6 | 1 |
|
||||
| @aarshkshah1992 | 1 | +6/-6 | 2 |
|
||||
| @dirkmc | 2 | +8/-0 | 2 |
|
||||
| @mx | 2 | +6/-1 | 2 |
|
||||
| @travisperson | 1 | +3/-2 | 1 |
|
||||
| @jennijuju | 2 | +2/-2 | 2 |
|
||||
| @ribasushi | 1 | +1/-2 | 2 |
|
||||
|
||||
# 1.11.1 / 2021-08-16
|
||||
|
||||
> Note: for discussion about this release, please comment [here](https://github.com/filecoin-project/lotus/discussions/6904)
|
||||
|
||||
This is a **highly recommended** but optional Lotus v1.11.1 release that introduces many deal making and datastore improvements and new features along with other bug fixes.
|
||||
|
||||
## Highlights
|
||||
- ⭐️⭐️⭐️[**lotus-miner market subsystem**](https://docs.filecoin.io/mine/lotus/split-markets-miners/#frontmatter-title) is introduced in this release! It is **highly recommended** for storage providers to run markets processes on a separate machine! Doing so, only this machine needs to exposes public ports for deal making. This also means that the other miner operations can now be completely isolated by from the deal making processes and storage providers can stop and restarts the markets process without affecting an ongoing Winning/Window PoSt!
|
||||
- More details on the concepts, architecture and how to split the market process can be found [here](https://docs.filecoin.io/mine/lotus/split-markets-miners/#concepts).
|
||||
- Base on your system setup(running on separate machines, same machine and so on), please see the suggested practice by community members [here](https://github.com/filecoin-project/lotus/discussions/7047#discussion-3515335).
|
||||
- Note: if you are running lotus-worker on a different machine, you will need to set `MARKETS_API_INFO` for certain CLI to work properly. This will be improved by #7072.
|
||||
- Huge thanks to MinerX fellows for [helping testing the implementation, reporting the issues so they were fixed by now and providing feedbacks](https://github.com/filecoin-project/lotus/discussions/6861) to user docs in the past three weeks!
|
||||
- Config for collateral from miner available balance ([filecoin-project/lotus#6629](https://github.com/filecoin-project/lotus/pull/6629))
|
||||
- Better control your sector collateral payment by setting `CollateralFromMinerBalance`, `AvailableBalanceBuffer` and `DisableCollateralFallback`.
|
||||
- `CollateralFromMinerBalance`: whether to use available miner balance for sector collateral instead of sending it with each message, default is `false`.
|
||||
- `AvailableBalanceBuffer`: minimum available balance to keep in the miner actor before sending it with messages, default is 0FIL.
|
||||
- `DisableCollateralFallback`: whether to send collateral with messages even if there is no available balance in the miner actor, default is `false`.
|
||||
- Config for deal publishing control addresses ([filecoin-project/lotus#6697](https://github.com/filecoin-project/lotus/pull/6697))
|
||||
- Set `DealPublishControl` to set the wallet used for sending `PublishStorageDeals` messages, instructions [here](https://docs.filecoin.io/mine/lotus/miner-addresses/#control-addresses).
|
||||
- Config UX improvements ([filecoin-project/lotus#6848](https://github.com/filecoin-project/lotus/pull/6848))
|
||||
- You can now preview the the default and updated node config by running `lotus/lotus-miner config default/updated`
|
||||
|
||||
## New Features
|
||||
- ⭐️⭐️⭐️ Support standalone miner-market process ([filecoin-project/lotus#6356](https://github.com/filecoin-project/lotus/pull/6356))
|
||||
- **⭐️⭐️ Experimental** [Splitstore]((https://github.com/filecoin-project/lotus/blob/master/blockstore/splitstore/README.md)) (more details coming in v1.11.2! Stay tuned! Join the discussion [here](https://github.com/filecoin-project/lotus/discussions/5788) if you have questions!) :
|
||||
- Improve splitstore warmup ([filecoin-project/lotus#6867](https://github.com/filecoin-project/lotus/pull/6867))
|
||||
- Moving GC for badger ([filecoin-project/lotus#6854](https://github.com/filecoin-project/lotus/pull/6854))
|
||||
- splitstore shed utils ([filecoin-project/lotus#6811](https://github.com/filecoin-project/lotus/pull/6811))
|
||||
- fix warmup by decoupling state from message receipt walk ([filecoin-project/lotus#6841](https://github.com/filecoin-project/lotus/pull/6841))
|
||||
- Splitstore: support on-disk marksets using badger ([filecoin-project/lotus#6833](https://github.com/filecoin-project/lotus/pull/6833))
|
||||
- cache loaded block messages ([filecoin-project/lotus#6760](https://github.com/filecoin-project/lotus/pull/6760))
|
||||
- Splitstore: add retention policy option for keeping messages in the hotstore ([filecoin-project/lotus#6775](https://github.com/filecoin-project/lotus/pull/6775))
|
||||
- Introduce the LOTUS_CHAIN_BADGERSTORE_DISABLE_FSYNC envvar ([filecoin-project/lotus#6817](https://github.com/filecoin-project/lotus/pull/6817))
|
||||
- Splitstore: add support for protecting out of chain references in the blockstore ([filecoin-project/lotus#6777](https://github.com/filecoin-project/lotus/pull/6777))
|
||||
- Implement exposed splitstore ([filecoin-project/lotus#6762](https://github.com/filecoin-project/lotus/pull/6762))
|
||||
- Splitstore code reorg ([filecoin-project/lotus#6756](https://github.com/filecoin-project/lotus/pull/6756))
|
||||
- Splitstore: Some small fixes ([filecoin-project/lotus#6754](https://github.com/filecoin-project/lotus/pull/6754))
|
||||
- Splitstore Enhanchements ([filecoin-project/lotus#6474](https://github.com/filecoin-project/lotus/pull/6474))
|
||||
- lotus-shed: initial export cmd for markets related metadata ([filecoin-project/lotus#6840](https://github.com/filecoin-project/lotus/pull/6840))
|
||||
- add a very verbose -vv flag to lotus and lotus-miner. ([filecoin-project/lotus#6888](https://github.com/filecoin-project/lotus/pull/6888))
|
||||
- Add allocated sectorid vis ([filecoin-project/lotus#4638](https://github.com/filecoin-project/lotus/pull/4638))
|
||||
- add a command for compacting sector numbers bitfield ([filecoin-project/lotus#4640](https://github.com/filecoin-project/lotus/pull/4640))
|
||||
- Run `lotus-miner actor compact-allocated` to compact sector number allocations to reduce the size of the allocated sector number bitfield.
|
||||
- Add ChainGetMessagesInTipset API ([filecoin-project/lotus#6642](https://github.com/filecoin-project/lotus/pull/6642))
|
||||
- Handle the --color flag via proper global state ([filecoin-project/lotus#6743](https://github.com/filecoin-project/lotus/pull/6743))
|
||||
- Enable color by default only if os.Stdout is a TTY ([filecoin-project/lotus#6696](https://github.com/filecoin-project/lotus/pull/6696))
|
||||
- Stop outputing ANSI color on non-TTY ([filecoin-project/lotus#6694](https://github.com/filecoin-project/lotus/pull/6694))
|
||||
- Envvar to disable slash filter ([filecoin-project/lotus#6620](https://github.com/filecoin-project/lotus/pull/6620))
|
||||
- commit batch: AggregateAboveBaseFee config ([filecoin-project/lotus#6650](https://github.com/filecoin-project/lotus/pull/6650))
|
||||
- shed tool to estimate aggregate network fees ([filecoin-project/lotus#6631](https://github.com/filecoin-project/lotus/pull/6631))
|
||||
|
||||
## Bug Fixes
|
||||
- Fix padding of deals, which only partially shipped in #5988 ([filecoin-project/lotus#6683](https://github.com/filecoin-project/lotus/pull/6683))
|
||||
- fix deal concurrency test failures by upgrading graphsync and others ([filecoin-project/lotus#6724](https://github.com/filecoin-project/lotus/pull/6724))
|
||||
- fix: on randomness change, use new rand ([filecoin-project/lotus#6805](https://github.com/filecoin-project/lotus/pull/6805)) - fix: always check if StateSearchMessage returns nil ([filecoin-project/lotus#6802](https://github.com/filecoin-project/lotus/pull/6802))
|
||||
- test: fix flaky window post tests ([filecoin-project/lotus#6804](https://github.com/filecoin-project/lotus/pull/6804))
|
||||
- wrap close(wait) with sync.Once to avoid panic ([filecoin-project/lotus#6800](https://github.com/filecoin-project/lotus/pull/6800))
|
||||
- fixes #6786 segfault ([filecoin-project/lotus#6787](https://github.com/filecoin-project/lotus/pull/6787))
|
||||
- ClientRetrieve stops on cancel([filecoin-project/lotus#6739](https://github.com/filecoin-project/lotus/pull/6739))
|
||||
- Fix bugs in sectors extend --v1-sectors ([filecoin-project/lotus#6066](https://github.com/filecoin-project/lotus/pull/6066))
|
||||
- fix "lotus-seed genesis car" error "merkledag: not found" ([filecoin-project/lotus#6688](https://github.com/filecoin-project/lotus/pull/6688))
|
||||
- Get retrieval pricing input should not error out on a deal state fetch ([filecoin-project/lotus#6679](https://github.com/filecoin-project/lotus/pull/6679))
|
||||
- Fix more CID double-encoding as hex ([filecoin-project/lotus#6680](https://github.com/filecoin-project/lotus/pull/6680))
|
||||
- storage: Fix FinalizeSector with sectors in stoage paths ([filecoin-project/lotus#6653](https://github.com/filecoin-project/lotus/pull/6653))
|
||||
- Fix tiny error in check-client-datacap ([filecoin-project/lotus#6664](https://github.com/filecoin-project/lotus/pull/6664))
|
||||
- Fix: precommit_batch method used the wrong cfg.CommitBatchWait ([filecoin-project/lotus#6658](https://github.com/filecoin-project/lotus/pull/6658))
|
||||
- fix ticket expiration check ([filecoin-project/lotus#6635](https://github.com/filecoin-project/lotus/pull/6635))
|
||||
- remove precommit check in handleCommitFailed ([filecoin-project/lotus#6634](https://github.com/filecoin-project/lotus/pull/6634))
|
||||
- fix prove commit aggregate send token amount ([filecoin-project/lotus#6625](https://github.com/filecoin-project/lotus/pull/6625))
|
||||
|
||||
## Improvements
|
||||
- Eliminate inefficiency in markets logging ([filecoin-project/lotus#6895](https://github.com/filecoin-project/lotus/pull/6895))
|
||||
- rename `cmd/lotus{-storage=>}-miner` to match binary. ([filecoin-project/lotus#6886](https://github.com/filecoin-project/lotus/pull/6886))
|
||||
- fix racy TestSimultanenousTransferLimit. ([filecoin-project/lotus#6862](https://github.com/filecoin-project/lotus/pull/6862))
|
||||
- ValidateBlock: Assert that block header height's are greater than parents ([filecoin-project/lotus#6872](https://github.com/filecoin-project/lotus/pull/6872))
|
||||
- feat: Don't panic when api impl is nil ([filecoin-project/lotus#6857](https://github.com/filecoin-project/lotus/pull/6857))
|
||||
- add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
|
||||
- easy way to make install app ([filecoin-project/lotus#5183](https://github.com/filecoin-project/lotus/pull/5183))
|
||||
- api: Separate the Net interface from Common ([filecoin-project/lotus#6627](https://github.com/filecoin-project/lotus/pull/6627)) - add StateReadState to gateway api ([filecoin-project/lotus#6818](https://github.com/filecoin-project/lotus/pull/6818))
|
||||
- add SealProof in SectorBuilder ([filecoin-project/lotus#6815](https://github.com/filecoin-project/lotus/pull/6815))
|
||||
- sealing: Handle preCommitParams errors more correctly ([filecoin-project/lotus#6763](https://github.com/filecoin-project/lotus/pull/6763))
|
||||
- ClientFindData: always fetch peer id from chain ([filecoin-project/lotus#6807](https://github.com/filecoin-project/lotus/pull/6807))
|
||||
- test: handle null blocks in TestForkRefuseCall ([filecoin-project/lotus#6758](https://github.com/filecoin-project/lotus/pull/6758))
|
||||
- Add more deal details to lotus-miner info ([filecoin-project/lotus#6708](https://github.com/filecoin-project/lotus/pull/6708))
|
||||
- add election backtest ([filecoin-project/lotus#5950](https://github.com/filecoin-project/lotus/pull/5950))
|
||||
- add dollar sign ([filecoin-project/lotus#6690](https://github.com/filecoin-project/lotus/pull/6690))
|
||||
- get-actor cli spelling fix ([filecoin-project/lotus#6681](https://github.com/filecoin-project/lotus/pull/6681))
|
||||
- polish(statetree): accept a context in statetree diff for timeouts ([filecoin-project/lotus#6639](https://github.com/filecoin-project/lotus/pull/6639))
|
||||
- Add helptext to lotus chain export ([filecoin-project/lotus#6672](https://github.com/filecoin-project/lotus/pull/6672))
|
||||
- add an incremental nonce itest. ([filecoin-project/lotus#6663](https://github.com/filecoin-project/lotus/pull/6663))
|
||||
- commit batch: Initialize the FailedSectors map ([filecoin-project/lotus#6647](https://github.com/filecoin-project/lotus/pull/6647))
|
||||
- Fast-path retry submitting commit aggregate if commit is still valid ([filecoin-project/lotus#6638](https://github.com/filecoin-project/lotus/pull/6638))
|
||||
- Reuse timers in sealing batch logic ([filecoin-project/lotus#6636](https://github.com/filecoin-project/lotus/pull/6636))
|
||||
|
||||
## Dependency Updates
|
||||
- Update to proof v8.0.3 ([filecoin-project/lotus#6890](https://github.com/filecoin-project/lotus/pull/6890))
|
||||
- update to go-fil-market v1.6.0 ([filecoin-project/lotus#6885](https://github.com/filecoin-project/lotus/pull/6885))
|
||||
- Bump go-multihash, adjust test for supported version ([filecoin-project/lotus#6674](https://github.com/filecoin-project/lotus/pull/6674))
|
||||
- github.com/filecoin-project/go-data-transfer (v1.6.0 -> v1.7.2):
|
||||
- github.com/filecoin-project/go-fil-markets (v1.5.0 -> v1.6.2):
|
||||
- github.com/filecoin-project/go-padreader (v0.0.0-20200903213702-ed5fae088b20 -> v0.0.0-20210723183308-812a16dc01b1)
|
||||
- github.com/filecoin-project/go-state-types (v0.1.1-0.20210506134452-99b279731c48 -> v0.1.1-0.20210810190654-139e0e79e69e)
|
||||
- github.com/filecoin-project/go-statemachine (v0.0.0-20200925024713-05bd7c71fbfe -> v1.0.1)
|
||||
- update go-libp2p-pubsub to v0.5.0 ([filecoin-project/lotus#6764](https://github.com/filecoin-project/lotus/pull/6764))
|
||||
|
||||
## Others
|
||||
- Master->v1.11.1 ([filecoin-project/lotus#7051](https://github.com/filecoin-project/lotus/pull/7051))
|
||||
- v1.11.1-rc2 ([filecoin-project/lotus#6966](https://github.com/filecoin-project/lotus/pull/6966))
|
||||
- Backport master -> v1.11.1 ([filecoin-project/lotus#6965](https://github.com/filecoin-project/lotus/pull/6965))
|
||||
- Fixes in master -> release ([filecoin-project/lotus#6933](https://github.com/filecoin-project/lotus/pull/6933))
|
||||
- Add changelog for v1.11.1-rc1 and bump the version ([filecoin-project/lotus#6900](https://github.com/filecoin-project/lotus/pull/6900))
|
||||
- Fix merge release -> v1.11.1 ([filecoin-project/lotus#6897](https://github.com/filecoin-project/lotus/pull/6897))
|
||||
- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6880](https://github.com/filecoin-project/lotus/pull/6880))
|
||||
- Add github actions for staled pr ([filecoin-project/lotus#6879](https://github.com/filecoin-project/lotus/pull/6879))
|
||||
- Update issue templates and add templates for M1 ([filecoin-project/lotus#6856](https://github.com/filecoin-project/lotus/pull/6856))
|
||||
- Fix links in issue templates
|
||||
- Update issue templates to forms ([filecoin-project/lotus#6798](https://github.com/filecoin-project/lotus/pull/6798)
|
||||
- Nerpa v13 upgrade ([filecoin-project/lotus#6837](https://github.com/filecoin-project/lotus/pull/6837))
|
||||
- add docker-compose file ([filecoin-project/lotus#6544](https://github.com/filecoin-project/lotus/pull/6544))
|
||||
- release -> master ([filecoin-project/lotus#6828](https://github.com/filecoin-project/lotus/pull/6828))
|
||||
- Resurrect CODEOWNERS, but for maintainers group ([filecoin-project/lotus#6773](https://github.com/filecoin-project/lotus/pull/6773))
|
||||
- Master disclaimer ([filecoin-project/lotus#6757](https://github.com/filecoin-project/lotus/pull/6757))
|
||||
- Create stale.yml ([filecoin-project/lotus#6747](https://github.com/filecoin-project/lotus/pull/6747))
|
||||
- Release template: Update all testnet infra at once ([filecoin-project/lotus#6710](https://github.com/filecoin-project/lotus/pull/6710))
|
||||
- Release Template: remove binary validation step ([filecoin-project/lotus#6709](https://github.com/filecoin-project/lotus/pull/6709))
|
||||
- Reset of the interop network ([filecoin-project/lotus#6689](https://github.com/filecoin-project/lotus/pull/6689))
|
||||
- Update version.go to 1.11.1 ([filecoin-project/lotus#6621](https://github.com/filecoin-project/lotus/pull/6621))
|
||||
|
||||
## Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @vyzo | 313 | +8928/-6010 | 415 |
|
||||
| @nonsense | 103 | +6041/-4041 | 304 |
|
||||
| @magik6k | 37 | +3851/-1611 | 146 |
|
||||
| @ZenGround0 | 24 | +1693/-1394 | 95 |
|
||||
| @placer14 | 1 | +2310/-578 | 8 |
|
||||
| @dirkmc | 7 | +1154/-726 | 29 |
|
||||
| @raulk | 44 | +969/-616 | 141 |
|
||||
| @jennijuju | 15 | +682/-354 | 47 |
|
||||
| @ribasushi | 18 | +469/-273 | 64 |
|
||||
| @coryschwartz | 5 | +576/-135 | 14 |
|
||||
| @hunjixin | 7 | +404/-82 | 19 |
|
||||
| @dirkmc | 17 | +348/-47 | 17 |
|
||||
| @tchardin | 2 | +262/-34 | 5 |
|
||||
| @aarshkshah1992 | 9 | +233/-63 | 44 |
|
||||
| @Kubuxu | 4 | +254/-16 | 4 |
|
||||
| @hannahhoward | 6 | +163/-75 | 8 |
|
||||
| @whyrusleeping | 4 | +157/-16 | 9 |
|
||||
| @Whyrusleeping | 2 | +87/-66 | 10 |
|
||||
| @arajasek | 10 | +81/-53 | 13 |
|
||||
| @zgfzgf | 2 | +104/-4 | 2 |
|
||||
| @aarshkshah1992 | 6 | +85/-19 | 10 |
|
||||
| @llifezou | 4 | +59/-20 | 4 |
|
||||
| @Stebalien | 7 | +47/-17 | 9 |
|
||||
| @johnli-helloworld | 3 | +46/-15 | 5 |
|
||||
| @frrist | 1 | +28/-23 | 2 |
|
||||
| @hannahhoward | 4 | +46/-5 | 11 |
|
||||
| @Jennifer | 4 | +31/-2 | 4 |
|
||||
| @wangchao | 1 | +1/-27 | 1 |
|
||||
| @jennijuju | 2 | +7/-21 | 2 |
|
||||
| @chadwick2143 | 1 | +15/-1 | 1 |
|
||||
| @Jerry | 2 | +9/-4 | 2 |
|
||||
| Steve Loeppky | 2 | +12/-0 | 2 |
|
||||
| David Dias | 1 | +9/-0 | 1 |
|
||||
| dependabot[bot] | 1 | +3/-3 | 1 |
|
||||
| zhoutian527 | 1 | +2/-2 | 1 |
|
||||
| xloem | 1 | +4/-0 | 1 |
|
||||
| @travisperson| 2 | +2/-2 | 3 |
|
||||
| Liviu Damian | 2 | +2/-2 | 2 |
|
||||
| @jimpick | 2 | +2/-2 | 2 |
|
||||
| Frank | 1 | +3/-0 | 1 |
|
||||
| turuslan | 1 | +1/-1 | 1 |
|
||||
| Kirk Baird | 1 | +0/-0 | 1 |
|
||||
|
||||
# 1.11.0 / 2021-07-22
|
||||
|
||||
This is a **highly recommended** release of Lotus that have many bug fixes, improvements and new features.
|
||||
|
||||
## Highlights
|
||||
- Miner SimultaneousTransfers config ([filecoin-project/lotus#6612](https://github.com/filecoin-project/lotus/pull/6612))
|
||||
- Set `SimultaneousTransfers` in lotus miner config to configure the maximum number of parallel online data transfers, including both storage and retrieval deals.
|
||||
- Dynamic Retrieval pricing ([filecoin-project/lotus#6175](https://github.com/filecoin-project/lotus/pull/6175))
|
||||
- Customize your retrieval ask price, see a quick tutorial [here](https://github.com/filecoin-project/lotus/discussions/6780).
|
||||
- Robust message management ([filecoin-project/lotus#5822](https://github.com/filecoin-project/lotus/pull/5822))
|
||||
- run `lotus mpool manage and follow the instructions!
|
||||
- Demo available at https://www.youtube.com/watch?v=QDocpLQjZgQ.
|
||||
- Add utils to use multisigs as miner owners ([filecoin-project/lotus#6490](https://github.com/filecoin-project/lotus/pull/6490))
|
||||
|
||||
## More New Features
|
||||
- feat: implement lotus-sim ([filecoin-project/lotus#6406](https://github.com/filecoin-project/lotus/pull/6406))
|
||||
- implement a command to export a car ([filecoin-project/lotus#6405](https://github.com/filecoin-project/lotus/pull/6405))
|
||||
- Add a command to get the fees of a deal ([filecoin-project/lotus#5307](https://github.com/filecoin-project/lotus/pull/5307))
|
||||
- run `lotus-shed market get-deal-fees`
|
||||
- Add a command to list retrievals ([filecoin-project/lotus#6337](https://github.com/filecoin-project/lotus/pull/6337))
|
||||
- run `lotus client list-retrievals`
|
||||
- lotus-gateway: add check command ([filecoin-project/lotus#6373](https://github.com/filecoin-project/lotus/pull/6373))
|
||||
- lotus-wallet: JWT Support ([filecoin-project/lotus#6360](https://github.com/filecoin-project/lotus/pull/6360))
|
||||
- Allow starting networks from arbitrary actor versions ([filecoin-project/lotus#6305](https://github.com/filecoin-project/lotus/pull/6305))
|
||||
- oh, snap! ([filecoin-project/lotus#6202](https://github.com/filecoin-project/lotus/pull/6202))
|
||||
- Add a shed util to count 64 GiB miner stats ([filecoin-project/lotus#6290](https://github.com/filecoin-project/lotus/pull/6290))
|
||||
- Introduce stateless offline dealflow, bypassing the FSM/deallists ([filecoin-project/lotus#5961](https://github.com/filecoin-project/lotus/pull/5961))
|
||||
- Transplant some useful commands to lotus-shed actor ([filecoin-project/lotus#5913](https://github.com/filecoin-project/lotus/pull/5913))
|
||||
- run `lotus-shed actor`
|
||||
- actor wrapper codegen ([filecoin-project/lotus#6108](https://github.com/filecoin-project/lotus/pull/6108))
|
||||
- Add a shed util to count miners by post type ([filecoin-project/lotus#6169](https://github.com/filecoin-project/lotus/pull/6169))
|
||||
- shed: command to list duplicate messages in tipsets (steb) ([filecoin-project/lotus#5847](https://github.com/filecoin-project/lotus/pull/5847))
|
||||
- feat: allow checkpointing to forks ([filecoin-project/lotus#6107](https://github.com/filecoin-project/lotus/pull/6107))
|
||||
- Add a CLI tool for miner proving deadline ([filecoin-project/lotus#6132](https://github.com/filecoin-project/lotus/pull/6132))
|
||||
- run `lotus state miner-proving-deadline`
|
||||
|
||||
|
||||
## Bug Fixes
|
||||
- Fix wallet error messages ([filecoin-project/lotus#6594](https://github.com/filecoin-project/lotus/pull/6594))
|
||||
- Fix CircleCI gen ([filecoin-project/lotus#6589](https://github.com/filecoin-project/lotus/pull/6589))
|
||||
- Make query-ask CLI more graceful ([filecoin-project/lotus#6590](https://github.com/filecoin-project/lotus/pull/6590))
|
||||
- scale up sector expiration to avoid sector expire in batch-pre-commit waitting ([filecoin-project/lotus#6566](https://github.com/filecoin-project/lotus/pull/6566))
|
||||
- Fix an error in msigLockCancel ([filecoin-project/lotus#6582](https://github.com/filecoin-project/lotus/pull/6582)
|
||||
- fix circleci being out of sync. ([filecoin-project/lotus#6573](https://github.com/filecoin-project/lotus/pull/6573))
|
||||
- Fix helptext for ask price([filecoin-project/lotus#6560](https://github.com/filecoin-project/lotus/pull/6560))
|
||||
- fix commit finalize failed ([filecoin-project/lotus#6521](https://github.com/filecoin-project/lotus/pull/6521))
|
||||
- Fix soup ([filecoin-project/lotus#6501](https://github.com/filecoin-project/lotus/pull/6501))
|
||||
- fix: pick the correct partitions-per-post limit ([filecoin-project/lotus#6502](https://github.com/filecoin-project/lotus/pull/6502))
|
||||
- sealing: Fix restartSectors race ([filecoin-project/lotus#6495](https://github.com/filecoin-project/lotus/pull/6495))
|
||||
- Fix: correct the change of message size limit ([filecoin-project/lotus#6430](https://github.com/filecoin-project/lotus/pull/6430))
|
||||
- Fix logging of stringified CIDs double-encoded in hex ([filecoin-project/lotus#6413](https://github.com/filecoin-project/lotus/pull/6413))
|
||||
- Fix success handling in Retreival ([filecoin-project/lotus#5921](https://github.com/filecoin-project/lotus/pull/5921))
|
||||
- storagefsm: Fix batch deal packing behavior ([filecoin-project/lotus#6041](https://github.com/filecoin-project/lotus/pull/6041))
|
||||
- events: Fix handling of multiple matched events per epoch ([filecoin-project/lotus#6355](https://github.com/filecoin-project/lotus/pull/6355))
|
||||
- Fix logging around mineOne ([filecoin-project/lotus#6310](https://github.com/filecoin-project/lotus/pull/6310))
|
||||
- Fix shell completions ([filecoin-project/lotus#6316](https://github.com/filecoin-project/lotus/pull/6316))
|
||||
- Allow 8MB sectors in devnet ([filecoin-project/lotus#6312](https://github.com/filecoin-project/lotus/pull/6312))
|
||||
- fix ticket expired ([filecoin-project/lotus#6304](https://github.com/filecoin-project/lotus/pull/6304))
|
||||
- Revert "chore: update go-libp2p" ([filecoin-project/lotus#6306](https://github.com/filecoin-project/lotus/pull/6306))
|
||||
- fix: wait-api should use GetAPI to acquire binary specific API ([filecoin-project/lotus#6246](https://github.com/filecoin-project/lotus/pull/6246))
|
||||
- fix(ci): Updates to lotus CI build process ([filecoin-project/lotus#6256](https://github.com/filecoin-project/lotus/pull/6256))
|
||||
- fix: use a consistent tipset in commands ([filecoin-project/lotus#6142](https://github.com/filecoin-project/lotus/pull/6142))
|
||||
- go mod tidy for lotus-soup testplans ([filecoin-project/lotus#6124](https://github.com/filecoin-project/lotus/pull/6124))
|
||||
- fix testground payment channel tests: use 1 miner ([filecoin-project/lotus#6126](https://github.com/filecoin-project/lotus/pull/6126))
|
||||
- fix: use the parent state when listing actors ([filecoin-project/lotus#6143](https://github.com/filecoin-project/lotus/pull/6143))
|
||||
- Speed up StateListMessages in some cases ([filecoin-project/lotus#6007](https://github.com/filecoin-project/lotus/pull/6007))
|
||||
- fix(splitstore): fix a panic on revert-only head changes ([filecoin-project/lotus#6133](https://github.com/filecoin-project/lotus/pull/6133))
|
||||
- drand: fix beacon cache ([filecoin-project/lotus#6164](https://github.com/filecoin-project/lotus/pull/6164))
|
||||
|
||||
## Improvements
|
||||
- gateway: Add support for Version method ([filecoin-project/lotus#6618](https://github.com/filecoin-project/lotus/pull/6618))
|
||||
- revamped integration test kit (aka. Operation Sparks Joy) ([filecoin-project/lotus#6329](https://github.com/filecoin-project/lotus/pull/6329))
|
||||
- move with changed name ([filecoin-project/lotus#6587](https://github.com/filecoin-project/lotus/pull/6587))
|
||||
- dynamic circleci config for streamlining test execution ([filecoin-project/lotus#6561](https://github.com/filecoin-project/lotus/pull/6561))
|
||||
- extern/storage: add ability to ignore worker resources when scheduling. ([filecoin-project/lotus#6542](https://github.com/filecoin-project/lotus/pull/6542))
|
||||
- Adjust various CLI display ratios to arbitrary precision ([filecoin-project/lotus#6309](https://github.com/filecoin-project/lotus/pull/6309))
|
||||
- Test multicore SDR support ([filecoin-project/lotus#6479](https://github.com/filecoin-project/lotus/pull/6479))
|
||||
- Unit tests for sector batchers ([filecoin-project/lotus#6432](https://github.com/filecoin-project/lotus/pull/6432))
|
||||
- Update chain list with correct help instructions ([filecoin-project/lotus#6465](https://github.com/filecoin-project/lotus/pull/6465))
|
||||
- clean failed sectors in batch commit ([filecoin-project/lotus#6451](https://github.com/filecoin-project/lotus/pull/6451))
|
||||
- itests/kit: add guard to ensure imports from tests only. ([filecoin-project/lotus#6445](https://github.com/filecoin-project/lotus/pull/6445))
|
||||
- consolidate integration tests into `itests` package; create test kit; cleanup ([filecoin-project/lotus#6311](https://github.com/filecoin-project/lotus/pull/6311))
|
||||
- Fee config for sector batching ([filecoin-project/lotus#6420](https://github.com/filecoin-project/lotus/pull/6420))
|
||||
- UX: lotus state power CLI should fail if called with a not-miner ([filecoin-project/lotus#6425](https://github.com/filecoin-project/lotus/pull/6425))
|
||||
- Increase message size limit ([filecoin-project/lotus#6419](https://github.com/filecoin-project/lotus/pull/6419))
|
||||
- polish(stmgr): define ExecMonitor for message application callback ([filecoin-project/lotus#6389](https://github.com/filecoin-project/lotus/pull/6389))
|
||||
- upgrade testground action version ([filecoin-project/lotus#6403](https://github.com/filecoin-project/lotus/pull/6403))
|
||||
- Bypass task scheduler for reading unsealed pieces ([filecoin-project/lotus#6280](https://github.com/filecoin-project/lotus/pull/6280))
|
||||
- testplans: lotus-soup: use default WPoStChallengeWindow ([filecoin-project/lotus#6400](https://github.com/filecoin-project/lotus/pull/6400))
|
||||
- Integration tests for offline deals ([filecoin-project/lotus#6081](https://github.com/filecoin-project/lotus/pull/6081))
|
||||
- Fix some flaky tests ([filecoin-project/lotus#6397](https://github.com/filecoin-project/lotus/pull/6397))
|
||||
- build appimage in CI ([filecoin-project/lotus#6384](https://github.com/filecoin-project/lotus/pull/6384))
|
||||
- Generate AppImage ([filecoin-project/lotus#6208](https://github.com/filecoin-project/lotus/pull/6208))
|
||||
- Add test for AddVerifiedClient ([filecoin-project/lotus#6317](https://github.com/filecoin-project/lotus/pull/6317))
|
||||
- Typo fix in error message: "pubusb" -> "pubsub" ([filecoin-project/lotus#6365](https://github.com/filecoin-project/lotus/pull/6365))
|
||||
- Improve the cli state call command ([filecoin-project/lotus#6226](https://github.com/filecoin-project/lotus/pull/6226))
|
||||
- Upscale mineOne message to a WARN on unexpected ineligibility ([filecoin-project/lotus#6358](https://github.com/filecoin-project/lotus/pull/6358))
|
||||
- Remove few useless variable assignments ([filecoin-project/lotus#6359](https://github.com/filecoin-project/lotus/pull/6359))
|
||||
- Reduce noise from 'peer has different genesis' messages ([filecoin-project/lotus#6357](https://github.com/filecoin-project/lotus/pull/6357))
|
||||
- Get current seal proof when necessary ([filecoin-project/lotus#6339](https://github.com/filecoin-project/lotus/pull/6339))
|
||||
- Remove log line when tracing is not configured ([filecoin-project/lotus#6334](https://github.com/filecoin-project/lotus/pull/6334))
|
||||
- separate tracing environment variables ([filecoin-project/lotus#6323](https://github.com/filecoin-project/lotus/pull/6323))
|
||||
- feat: log dispute rate ([filecoin-project/lotus#6322](https://github.com/filecoin-project/lotus/pull/6322))
|
||||
- Move verifreg shed utils to CLI ([filecoin-project/lotus#6135](https://github.com/filecoin-project/lotus/pull/6135))
|
||||
- consider storiface.PathStorage when calculating storage requirements ([filecoin-project/lotus#6233](https://github.com/filecoin-project/lotus/pull/6233))
|
||||
- `storage` module: add go docs and minor code quality refactors ([filecoin-project/lotus#6259](https://github.com/filecoin-project/lotus/pull/6259))
|
||||
- Increase data transfer timeouts ([filecoin-project/lotus#6300](https://github.com/filecoin-project/lotus/pull/6300))
|
||||
- gateway: spin off from cmd to package ([filecoin-project/lotus#6294](https://github.com/filecoin-project/lotus/pull/6294))
|
||||
- Return total power when GetPowerRaw doesn't find miner claim ([filecoin-project/lotus#4938](https://github.com/filecoin-project/lotus/pull/4938))
|
||||
- add flags to control gateway lookback parameters ([filecoin-project/lotus#6247](https://github.com/filecoin-project/lotus/pull/6247))
|
||||
- chore(ci): Enable build on RC tags ([filecoin-project/lotus#6238](https://github.com/filecoin-project/lotus/pull/6238))
|
||||
- cron-wc ([filecoin-project/lotus#6178](https://github.com/filecoin-project/lotus/pull/6178))
|
||||
- Allow creation of state tree v3s ([filecoin-project/lotus#6167](https://github.com/filecoin-project/lotus/pull/6167))
|
||||
- mpool: Cleanup pre-nv12 selection logic ([filecoin-project/lotus#6148](https://github.com/filecoin-project/lotus/pull/6148))
|
||||
- attempt to do better padding on pieces being written into sectors ([filecoin-project/lotus#5988](https://github.com/filecoin-project/lotus/pull/5988))
|
||||
- remove duplicate ask and calculate ping before lock ([filecoin-project/lotus#5968](https://github.com/filecoin-project/lotus/pull/5968))
|
||||
- flaky tests improvement: separate TestBatchDealInput from TestAPIDealFlow ([filecoin-project/lotus#6141](https://github.com/filecoin-project/lotus/pull/6141))
|
||||
- Testground checks on push ([filecoin-project/lotus#5887](https://github.com/filecoin-project/lotus/pull/5887))
|
||||
- Use EmptyTSK where appropriate ([filecoin-project/lotus#6134](https://github.com/filecoin-project/lotus/pull/6134))
|
||||
- upgrade `lotus-soup` testplans and reduce deals concurrency to a single miner ([filecoin-project/lotus#6122](https://github.com/filecoin-project/lotus/pull/6122)
|
||||
|
||||
## Dependency Updates
|
||||
- downgrade libp2p/go-libp2p-yamux to v0.5.1. ([filecoin-project/lotus#6605](https://github.com/filecoin-project/lotus/pull/6605))
|
||||
- Update libp2p to 0.14.2 ([filecoin-project/lotus#6404](https://github.com/filecoin-project/lotus/pull/6404))
|
||||
- update to markets-v1.4.0 ([filecoin-project/lotus#6369](https://github.com/filecoin-project/lotus/pull/6369))
|
||||
- Use new actor tags ([filecoin-project/lotus#6291](https://github.com/filecoin-project/lotus/pull/6291))
|
||||
- chore: update go-libp2p ([filecoin-project/lotus#6231](https://github.com/filecoin-project/lotus/pull/6231))
|
||||
- Update ffi to proofs v7 ([filecoin-project/lotus#6150](https://github.com/filecoin-project/lotus/pull/6150))
|
||||
|
||||
## Others
|
||||
- Initial draft: basic build instructions on Readme ([filecoin-project/lotus#6498](https://github.com/filecoin-project/lotus/pull/6498))
|
||||
- Remove rc changelog, compile the new changelog for final release only ([filecoin-project/lotus#6444](https://github.com/filecoin-project/lotus/pull/6444))
|
||||
- updated configuration comments for docs ([filecoin-project/lotus#6440](https://github.com/filecoin-project/lotus/pull/6440))
|
||||
- Set ntwk v13 HyperDrive Calibration upgrade epoch ([filecoin-project/lotus#6441](https://github.com/filecoin-project/lotus/pull/6441))
|
||||
- build snapcraft ([filecoin-project/lotus#6388](https://github.com/filecoin-project/lotus/pull/6388))
|
||||
- Fix the doc errors of the sealing config funcs ([filecoin-project/lotus#6399](https://github.com/filecoin-project/lotus/pull/6399))
|
||||
- Add doc on gas balancing ([filecoin-project/lotus#6392](https://github.com/filecoin-project/lotus/pull/6392))
|
||||
- Add interop network ([filecoin-project/lotus#6387](https://github.com/filecoin-project/lotus/pull/6387))
|
||||
- Network version 13 (v1.11) ([filecoin-project/lotus#6342](https://github.com/filecoin-project/lotus/pull/6342))
|
||||
- Add a warning to the release issue template ([filecoin-project/lotus#6374](https://github.com/filecoin-project/lotus/pull/6374))
|
||||
- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6236](https://github.com/filecoin-project/lotus/pull/6236))
|
||||
- Delete CODEOWNERS ([filecoin-project/lotus#6289](https://github.com/filecoin-project/lotus/pull/6289))
|
||||
- Feat/nerpa v4 ([filecoin-project/lotus#6248](https://github.com/filecoin-project/lotus/pull/6248))
|
||||
- Introduce a release issue template ([filecoin-project/lotus#5826](https://github.com/filecoin-project/lotus/pull/5826))
|
||||
- This is a 1:1 forward-port of PR#6183 from 1.9.x to master ([filecoin-project/lotus#6196](https://github.com/filecoin-project/lotus/pull/6196))
|
||||
- Update cli gen ([filecoin-project/lotus#6155](https://github.com/filecoin-project/lotus/pull/6155))
|
||||
- Generate CLI docs ([filecoin-project/lotus#6145](https://github.com/filecoin-project/lotus/pull/6145))
|
||||
|
||||
## Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @raulk | 118 | +11972/-10860 | 472 |
|
||||
| @magik6k | 65 | +10824/-4158 | 353 |
|
||||
| @aarshkshah1992 | 59 | +8057/-3355 | 224 |
|
||||
| @arajasek | 41 | +8786/-1691 | 331 |
|
||||
| @Stebalien | 106 | +7653/-2718 | 273 |
|
||||
| dirkmc | 11 | +2580/-1371 | 77 |
|
||||
| @dirkmc | 39 | +1865/-1194 | 79 |
|
||||
| @Kubuxu | 19 | +1973/-485 | 81 |
|
||||
| @vyzo | 4 | +1748/-330 | 50 |
|
||||
| @aarshkshah1992 | 5 | +1462/-213 | 27 |
|
||||
| @coryschwartz | 35 | +568/-206 | 59 |
|
||||
| @chadwick2143 | 3 | +739/-1 | 4 |
|
||||
| @ribasushi | 21 | +487/-164 | 36 |
|
||||
| @hannahhoward | 5 | +544/-5 | 19 |
|
||||
| @jennijuju | 9 | +241/-174 | 19 |
|
||||
| @frrist | 1 | +137/-88 | 7 |
|
||||
| @travisperson | 3 | +175/-6 | 7 |
|
||||
| @wadeAlexC | 1 | +48/-129 | 1 |
|
||||
| @whyrusleeping | 8 | +161/-13 | 11 |
|
||||
| lotus | 1 | +114/-46 | 1 |
|
||||
| @nonsense | 8 | +107/-53 | 20 |
|
||||
| @rjan90 | 4 | +115/-33 | 4 |
|
||||
| @ZenGround0 | 3 | +114/-1 | 4 |
|
||||
| @Aloxaf | 1 | +43/-61 | 7 |
|
||||
| @yaohcn | 4 | +89/-9 | 5 |
|
||||
| @mitchellsoo | 1 | +51/-0 | 1 |
|
||||
| @placer14 | 3 | +28/-18 | 4 |
|
||||
| @jennijuju | 6 | +9/-14 | 6 |
|
||||
| @Frank | 2 | +11/-10 | 2 |
|
||||
| @wangchao | 3 | +5/-4 | 4 |
|
||||
| @Steve Loeppky | 1 | +7/-1 | 1 |
|
||||
| @Lion | 1 | +4/-2 | 1 |
|
||||
| @Mimir | 1 | +2/-2 | 1 |
|
||||
| @raulk | 1 | +1/-1 | 1 |
|
||||
| @Jack Yao | 1 | +1/-1 | 1 |
|
||||
| @IPFSUnion | 1 | +1/-1 | 1 |
|
||||
|
||||
# 1.10.1 / 2021-07-05
|
||||
|
||||
This is an optional but **highly recommended** release of Lotus for lotus miners that has many bug fixes and improvements based on the feedback we got from the community since HyperDrive.
|
||||
|
||||
## New Features
|
||||
- commit batch: AggregateAboveBaseFee config #6650
|
||||
- `AggregateAboveBaseFee` is added to miner sealing configuration for setting the network base fee to start aggregating proofs. When the network base fee is lower than this value, the prove commits will be submitted individually via `ProveCommitSector`. According to the [Batch Incentive Alignment](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md#batch-incentive-alignment) introduced in FIP-0013, we recommend miners to set this value to 0.15 nanoFIL(which is the default value) to avoid unexpected aggregation fee in burn and enjoy the most benefits of aggregation!
|
||||
|
||||
## Bug Fixes
|
||||
- storage: Fix FinalizeSector with sectors in storage paths #6652
|
||||
- Fix tiny error in check-client-datacap #6664
|
||||
- Fix: precommit_batch method used the wrong cfg.PreCommitBatchWait #6658
|
||||
- to optimize the batchwait #6636
|
||||
- fix getTicket: sector precommitted but expired case #6635
|
||||
- handleSubmitCommitAggregate() exception handling #6595
|
||||
- remove precommit check in handleCommitFailed #6634
|
||||
- ensure agg fee is adequate
|
||||
- fix: miner balance is not enough, so that ProveCommitAggregate msg exec failed #6623
|
||||
- commit batch: Initialize the FailedSectors map #6647
|
||||
|
||||
Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @magik6k| 7 | +151/-56 | 21 |
|
||||
| @llifezou | 4 | +59/-20 | 4 |
|
||||
| @johnli-helloworld | 2 | +45/-14 | 4 |
|
||||
| @wangchao | 1 | +1/-27 | 1 |
|
||||
| Jerry | 2 | +9/-4 | 2 |
|
||||
| @zhoutian527 | 1 | +2/-2 | 1 |
|
||||
| @ribasushi| 1 | +1/-1 | 1 |
|
||||
|
||||
# 1.10.1 / 2021-07-05
|
||||
|
||||
This is an optional but **highly recommended** release of Lotus for lotus miners that has many bug fixes and improvements based on the feedback we got from the community since HyperDrive.
|
||||
|
||||
## New Features
|
||||
- commit batch: AggregateAboveBaseFee config #6650
|
||||
- `AggregateAboveBaseFee` is added to miner sealing configuration for setting the network base fee to start aggregating proofs. When the network base fee is lower than this value, the prove commits will be submitted individually via `ProveCommitSector`. According to the [Batch Incentive Alignment](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md#batch-incentive-alignment) introduced in FIP-0013, we recommend miners to set this value to 0.15 nanoFIL(which is the default value) to avoid unexpected aggregation fee in burn and enjoy the most benefits of aggregation!
|
||||
|
||||
## Bug Fixes
|
||||
- storage: Fix FinalizeSector with sectors in storage paths #6652
|
||||
- Fix tiny error in check-client-datacap #6664
|
||||
- Fix: precommit_batch method used the wrong cfg.PreCommitBatchWait #6658
|
||||
- to optimize the batchwait #6636
|
||||
- fix getTicket: sector precommitted but expired case #6635
|
||||
- handleSubmitCommitAggregate() exception handling #6595
|
||||
- remove precommit check in handleCommitFailed #6634
|
||||
- ensure agg fee is adequate
|
||||
- fix: miner balance is not enough, so that ProveCommitAggregate msg exec failed #6623
|
||||
- commit batch: Initialize the FailedSectors map #6647
|
||||
|
||||
Contributors
|
||||
|
||||
| Contributor | Commits | Lines ± | Files Changed |
|
||||
|-------------|---------|---------|---------------|
|
||||
| @magik6k| 7 | +151/-56 | 21 |
|
||||
| @llifezou | 4 | +59/-20 | 4 |
|
||||
| @johnli-helloworld | 2 | +45/-14 | 4 |
|
||||
| @wangchao | 1 | +1/-27 | 1 |
|
||||
| Jerry | 2 | +9/-4 | 2 |
|
||||
| @zhoutian527 | 1 | +2/-2 | 1 |
|
||||
| @ribasushi| 1 | +1/-1 | 1 |
|
||||
|
||||
# 1.10.0 / 2021-06-23
|
||||
|
||||
This is a mandatory release of Lotus that introduces Filecoin network v13, codenamed the HyperDrive upgrade. The
|
||||
|
||||
+8
-162
@@ -36,7 +36,7 @@ WORKDIR /opt/filecoin
|
||||
ARG RUSTFLAGS=""
|
||||
ARG GOFLAGS=""
|
||||
|
||||
RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
|
||||
RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats
|
||||
|
||||
|
||||
FROM ubuntu:20.04 AS base
|
||||
@@ -56,173 +56,19 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
|
||||
RUN useradd -r -u 532 -U fc
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY scripts/docker-lotus-entrypoint.sh /
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
|
||||
ENV DOCKER_LOTUS_IMPORT_WALLET ""
|
||||
|
||||
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
|
||||
ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
###
|
||||
FROM base AS lotus-wallet
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
|
||||
ENV WALLET_PATH /var/lib/lotus-wallet
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1777
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-wallet"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
###
|
||||
FROM base AS lotus-gateway
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-gateway"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus-miner
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY scripts/docker-lotus-miner-entrypoint.sh /
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV DOCKER_LOTUS_MINER_INIT true
|
||||
|
||||
RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 2345
|
||||
|
||||
ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus-worker
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
|
||||
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
|
||||
VOLUME /var/lib/lotus-worker
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 3456
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-worker"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
from base as lotus-all-in-one
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
|
||||
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
|
||||
ENV WALLET_PATH /var/lib/lotus-wallet
|
||||
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
|
||||
ENV DOCKER_LOTUS_MINER_INIT true
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
|
||||
RUN mkdir /var/tmp/filecoin-proof-parameters
|
||||
RUN mkdir /var/lib/lotus
|
||||
RUN mkdir /var/lib/lotus-miner
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus
|
||||
RUN chown fc: /var/lib/lotus-miner
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
|
||||
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/lib/lotus-worker
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
EXPOSE 2345
|
||||
EXPOSE 3456
|
||||
EXPOSE 1777
|
||||
ENTRYPOINT ["/usr/local/bin/lotus"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
|
||||
<!-- TOC -->
|
||||
|
||||
- [`lotus` Release Flow](#lotus-release-flow)
|
||||
- [Purpose](#purpose)
|
||||
- [High-level Summary](#high-level-summary)
|
||||
- [Motivation and Requirements](#motivation-and-requirements)
|
||||
- [Adopted Conventions](#adopted-conventions)
|
||||
- [Major Releases](#major-releases)
|
||||
- [Mandatory Releases](#mandatory-releases)
|
||||
- [Feature Releases](#feature-releases)
|
||||
- [Examples Scenarios](#examples-scenarios)
|
||||
- [Release Cycle](#release-cycle)
|
||||
- [Patch Releases](#patch-releases)
|
||||
- [Performing a Release](#performing-a-release)
|
||||
- [Security Fix Policy](#security-fix-policy)
|
||||
- [FAQ](#faq)
|
||||
- [Why aren't Go major versions used more?](#why-arent-go-major-versions-used-more)
|
||||
- [Related Items](#related-items)
|
||||
|
||||
<!-- /TOC -->
|
||||
|
||||
# `lotus` Release Flow
|
||||
|
||||
## Purpose
|
||||
|
||||
This document aims to describe how the Lotus team plans to ship releases of the Lotus implementation of Filecoin network. Interested parties can expect new releases to come out as described in this document.
|
||||
|
||||
## High-level Summary
|
||||
|
||||
- **Major releases** (1.0.0, 2.0.0, etc.) are reserved for significant changes to the Filecoin protocol that transform the network and its usecases. Such changes could include the addition of new Virtual Machines to the protocol, major upgrades to the Proofs of Replication used by Filecoin, etc.
|
||||
- Even minor releases (1.2.0, 1.4.0, etc.) of the Lotus software correspond to **mandatory releases** as they ship Filecoin network upgrades. Users **must** upgrade to these releases before a certain time in order to keep in sync with the Filecoin network. We aim to ensure there is at least one week before the upgrade deadline.
|
||||
- Patch versions of even minor releases (1.2.1, 1.4.2, etc.) correspond to **hotfix releases**. Such releases will only be shipped when a critical fix is needed to be applied on top of a mandatory release.
|
||||
- Odd minor releases (1.3.0, 1.5.0, etc.), as well as patch releases in these series (1.3.1, 1.5.2, etc.) correspond to **feature releases** with new development and bugfixes. These releases are not mandatory, but still highly recommended, **as they may contain critical security fixes**.
|
||||
- We aim to ship a new feature release of the Lotus software every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule.
|
||||
|
||||
## Motivation and Requirements
|
||||
|
||||
Our primary motivation is for users of the Lotus software (storage providers, storage clients, developers, etc.) to have a clear idea about when they can expect Lotus releases, and what they can expect in a release.
|
||||
|
||||
In order to achieve this, we need the following from our release process and conventions:
|
||||
|
||||
- Lotus version conventions make it immediately obvious whether a new Lotus release is mandatory or not. A release is mandatory if it ships a network upgrade to the Filecoin protocol.
|
||||
- The ability to ship critical fixes on top of mandatory releases, so as to avoid forcing users to consume larger unrelated changes.
|
||||
- A regular cadence of feature releases, so that users can know when a new Lotus release will be available for consumption.
|
||||
- The ability to ship any number of feature releases between two mandatory releases.
|
||||
- A clear description of the various stages of testing that a Lotus Release Candidate (RC) goes through.
|
||||
- Lotus Release issues will present a single source of truth for what may be contained in Lotus releases, including security fixes, and how they will be disclosed.
|
||||
|
||||
## Adopted Conventions
|
||||
|
||||
This section describes the conventions we have adopted. Users of Lotus are encouraged to review this in detail so as to be informed when new Lotus releases are shipped.
|
||||
|
||||
### Major Releases
|
||||
|
||||
Bumps to the Lotus major version number (1.0.0, 2.0.0, etc.) are reserved for significant changes to the Filecoin protocol that dramatically transform the network itself. Such changes could include the addition of new Virtual Machines to the protocol, major upgrades to the Proofs of Replication used by Filecoin, etc. These releases are expected to take lots of time to develop and will be rare. See also "Why aren't Go major versions used more?" below.
|
||||
|
||||
### Mandatory Releases
|
||||
|
||||
Even bumps to the Lotus minor version number (1.2.0, 1.4.0, etc.) are reserved for **mandatory releases** that ship Filecoin network upgrades. Users **must** upgrade to these releases before a certain time in order to keep in sync with the Filecoin network.
|
||||
|
||||
Depending on the scope of the upgrade, these releases may take up to several weeks to fully develop and test. We aim to ensure there are at least 2 weeks between the publication of the final Lotus release and the Filecoin network upgrade deadline.
|
||||
|
||||
These releases do not follow a regular cadence, as they are developed in lockstep with the other implementations of the Filecoin protocol. As of August 2021, the developers aim to ship 3-4 Filecoin network upgrades a year, though smaller security-critical upgrades may occur unpredictably.
|
||||
|
||||
Mandatory releases are somewhat sensitive since all Lotus users are forced to upgrade to them at the same time. As a result, they will be shipped on top of the most recent stable release of Lotus, which will generally be the latest Lotus release that has been in production for more than 2 weeks. (Note: given this rule, the basis of a mandatory release could be a mandatory release or a feature release depending on timing). Mandatory releases will not include any new feature development or bugfixes that haven't already baked in production for 2+ weeks, except for the changes needed for the network upgrade itself. Further, any critical fixes that are needed after the network upgrade will be shipped as patch version bumps to the mandatory release (1.2.1, 1.2.2, etc.) This prevents users from being forced to quickly digest unnecessary changes.
|
||||
|
||||
Users should generally aim to always upgrade to a new even minor version release since they either introduce a mandatory network upgrade or a critical fix.
|
||||
|
||||
### Feature Releases
|
||||
|
||||
All releases under an odd minor version number indicate **feature releases**. These could include releases such as 1.3.0, 1.3.1, 1.5.2, etc.
|
||||
|
||||
Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes. There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
|
||||
|
||||
We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the Release Cycle section (TODO: Link).
|
||||
|
||||
### Examples Scenarios
|
||||
|
||||
- **Scenario 1**: **Lotus 1.12.0 shipped a network upgrade, and no network upgrades are needed for a long while.**
|
||||
|
||||
In this case, the next feature release will be Lotus 1.13.0. In three-week intervals, we will ship Lotus 1.13.1, 1.13.2, and 1.13.3, all containing new features and bug fixes.
|
||||
|
||||
Let us assume that after the release of 1.13.3, a critical issue is discovered and a hotfix quickly developed. This hotfix will then be shipped in **both** 1.12.1 and 1.13.4. Users who have already upgrade to the 1.13 series can simply upgrade to 1.13.4. Users who have chosen to still be on 1.12.0, however, can use 1.12.1 to patch the critical issue without being forced to consume all the changes in the 1.13 series.
|
||||
|
||||
- **Scenario 2**: **Lotus 1.12.0 shipped a network upgrade, but the need for an unexpected network upgrade soon arises**
|
||||
|
||||
In this case, the Lotus 1.13 series will be dropped entirely, including any RCs that may have been undergoing testing. Instead, the network upgrade will be shipped as Lotus 1.14.0, built on top of Lotus 1.12.0. It will thus include no unnecessary changes, only the work needed to support the new network upgrade.
|
||||
|
||||
Any changes that were being worked on in the 1.13.0 series will then get applied on top of Lotus 1.14.0 and get shipped as Lotus 1.15.0.
|
||||
|
||||
## Release Cycle
|
||||
|
||||
A mandatory release process should take about 3-6 weeks, depending on the amount and the overall complexity of new features being introduced to the network protocol. It may also be shorter if there is a network incident that requires an emergency upgrade. A feature release process should take about 2-3 weeks.
|
||||
|
||||
The start time of the mandatory release process is subject to the network upgrade timeline. We will start a new feature release process every 3 weeks on Tuesdays, regardless of when the previous release landed unless it's still ongoing.
|
||||
|
||||
### Patch Releases
|
||||
|
||||
**Mandatory Release**
|
||||
|
||||
If we encounter a serious bug in a mandatory release post a network upgrade, we will create a patch release based on this release. Strictly only the fix to the bug will be included in the patch, and the bug fix will be backported to the master (dev) branch, and any ongoing feature release branch if applicable.
|
||||
|
||||
Patch release process for the mandatory releases will follow a compressed release cycle from hours to days depending on the severity and the impact to the network of the bug. In a patch release:
|
||||
|
||||
1. Automated and internal testing (stage 0 and 1) will be compressed into a few hours.
|
||||
2. Stage 2-3 will be skipped or shortened case by case.
|
||||
|
||||
Some patch releases, especially ones fixing one or more complex bugs that doesn't require a follow-up mandatory upgrade, may undergo the full release process.
|
||||
|
||||
**Feature Release**
|
||||
|
||||
Patch releases in odd minor releases (1.3.0, 1.5.0, etc.) like 1.3.1, 1.5.2 and etc are corresponding to another **feature releases** with new development and bugfixes. These releases are not mandatory, but still **highly** recommended, **as they may contain critical security fixes**.
|
||||
|
||||
---
|
||||
|
||||
### Performing a Release
|
||||
|
||||
At the beginning of each release cycle, we will generate our "Release tracking issue", which is populated with the content at [https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md)
|
||||
|
||||
This template will be used to track major goals we have, a planned shipping date, and a complete release checklist tied to a specific release.
|
||||
|
||||
### Security Fix Policy
|
||||
|
||||
Any release may contain security fixes. Unless the fix addresses a bug being exploited in the wild, the fix will *not* be called out in the release notes. Please make sure to update ASAP.
|
||||
|
||||
By policy, the team will usually wait until about 3 weeks after the final release to announce any fixed security issues. However, depending on the impact and ease of discovery of the issue, the team may wait more or less time.
|
||||
|
||||
It is important to always update to the latest version ASAP and file issues if you're unable to update for some reason.
|
||||
|
||||
Finally, unless a security issue is actively being exploited or a significant number of users are unable to update to the latest version (e.g., due to a difficult migration, breaking changes, etc.), security fixes will *not* be backported to previous releases.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Why aren't Go major versions used more?
|
||||
|
||||
Golang tightly couples source code with versioning (major versions beyond v1 leak into import paths). This poses logistical difficulties to using major versions here. Concretely, if we were to pick a policy that bumped the major version on every network upgrade, we would disrupt every single downstream library/application that consumed the native Lotus API (e.g., libraries depending on the JSON-RPC client, testground tests). They would need to update their code every single time that we released a network breaking change, even if it brought on zero expectation of breakage for the Golang APIs that they depend on. In this scenario, we are signaling breakage on the wrong API surface! We're signaling breakage on the Go level, when what breaks is the network protocol.
|
||||
|
||||
## Related Items
|
||||
|
||||
1. [Release Issue template](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md)
|
||||
2. [Lotus Release Flow Discussion](https://github.com/filecoin-project/lotus/discussions/7053): Leave a comment if you have any questions or feedbacks with regard to the lotus release flow.
|
||||
@@ -5,9 +5,7 @@ all: build
|
||||
|
||||
unexport GOFLAGS
|
||||
|
||||
GOCC?=go
|
||||
|
||||
GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
|
||||
GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
|
||||
ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
|
||||
$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
|
||||
$(error Update Golang to version to at least 1.16.0)
|
||||
@@ -87,32 +85,32 @@ interopnet: build-devnets
|
||||
|
||||
lotus: $(BUILD_DEPS)
|
||||
rm -f lotus
|
||||
$(GOCC) build $(GOFLAGS) -o lotus ./cmd/lotus
|
||||
go build $(GOFLAGS) -o lotus ./cmd/lotus
|
||||
|
||||
.PHONY: lotus
|
||||
BINS+=lotus
|
||||
|
||||
lotus-miner: $(BUILD_DEPS)
|
||||
rm -f lotus-miner
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
|
||||
go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
|
||||
.PHONY: lotus-miner
|
||||
BINS+=lotus-miner
|
||||
|
||||
lotus-worker: $(BUILD_DEPS)
|
||||
rm -f lotus-worker
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
|
||||
go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
|
||||
.PHONY: lotus-worker
|
||||
BINS+=lotus-worker
|
||||
|
||||
lotus-shed: $(BUILD_DEPS)
|
||||
rm -f lotus-shed
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
|
||||
go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
|
||||
.PHONY: lotus-shed
|
||||
BINS+=lotus-shed
|
||||
|
||||
lotus-gateway: $(BUILD_DEPS)
|
||||
rm -f lotus-gateway
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
|
||||
go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
|
||||
.PHONY: lotus-gateway
|
||||
BINS+=lotus-gateway
|
||||
|
||||
@@ -133,26 +131,23 @@ install-miner:
|
||||
install-worker:
|
||||
install -C ./lotus-worker /usr/local/bin/lotus-worker
|
||||
|
||||
install-app:
|
||||
install -C ./$(APP) /usr/local/bin/$(APP)
|
||||
|
||||
# TOOLS
|
||||
|
||||
lotus-seed: $(BUILD_DEPS)
|
||||
rm -f lotus-seed
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
|
||||
go build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
|
||||
|
||||
.PHONY: lotus-seed
|
||||
BINS+=lotus-seed
|
||||
|
||||
benchmarks:
|
||||
$(GOCC) run github.com/whyrusleeping/bencher ./... > bench.json
|
||||
go run github.com/whyrusleeping/bencher ./... > bench.json
|
||||
@echo Submitting results
|
||||
@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
|
||||
.PHONY: benchmarks
|
||||
|
||||
lotus-pond: 2k
|
||||
$(GOCC) build -o lotus-pond ./lotuspond
|
||||
go build -o lotus-pond ./lotuspond
|
||||
.PHONY: lotus-pond
|
||||
BINS+=lotus-pond
|
||||
|
||||
@@ -163,64 +158,85 @@ lotus-pond-front:
|
||||
lotus-pond-app: lotus-pond-front lotus-pond
|
||||
.PHONY: lotus-pond-app
|
||||
|
||||
lotus-townhall:
|
||||
rm -f lotus-townhall
|
||||
go build -o lotus-townhall ./cmd/lotus-townhall
|
||||
.PHONY: lotus-townhall
|
||||
BINS+=lotus-townhall
|
||||
|
||||
lotus-townhall-front:
|
||||
(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
|
||||
.PHONY: lotus-townhall-front
|
||||
|
||||
lotus-townhall-app: lotus-touch lotus-townhall-front
|
||||
.PHONY: lotus-townhall-app
|
||||
|
||||
lotus-fountain:
|
||||
rm -f lotus-fountain
|
||||
go build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
|
||||
go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
|
||||
go build -o lotus-fountain ./cmd/lotus-fountain
|
||||
.PHONY: lotus-fountain
|
||||
BINS+=lotus-fountain
|
||||
|
||||
lotus-chainwatch:
|
||||
rm -f lotus-chainwatch
|
||||
go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
|
||||
.PHONY: lotus-chainwatch
|
||||
BINS+=lotus-chainwatch
|
||||
|
||||
lotus-bench:
|
||||
rm -f lotus-bench
|
||||
$(GOCC) build -o lotus-bench ./cmd/lotus-bench
|
||||
go build -o lotus-bench ./cmd/lotus-bench
|
||||
.PHONY: lotus-bench
|
||||
BINS+=lotus-bench
|
||||
|
||||
lotus-stats:
|
||||
rm -f lotus-stats
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
|
||||
go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
|
||||
.PHONY: lotus-stats
|
||||
BINS+=lotus-stats
|
||||
|
||||
lotus-pcr:
|
||||
rm -f lotus-pcr
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
|
||||
go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
|
||||
.PHONY: lotus-pcr
|
||||
BINS+=lotus-pcr
|
||||
|
||||
lotus-health:
|
||||
rm -f lotus-health
|
||||
$(GOCC) build -o lotus-health ./cmd/lotus-health
|
||||
go build -o lotus-health ./cmd/lotus-health
|
||||
.PHONY: lotus-health
|
||||
BINS+=lotus-health
|
||||
|
||||
lotus-wallet:
|
||||
rm -f lotus-wallet
|
||||
$(GOCC) build -o lotus-wallet ./cmd/lotus-wallet
|
||||
go build -o lotus-wallet ./cmd/lotus-wallet
|
||||
.PHONY: lotus-wallet
|
||||
BINS+=lotus-wallet
|
||||
|
||||
lotus-keygen:
|
||||
rm -f lotus-keygen
|
||||
$(GOCC) build -o lotus-keygen ./cmd/lotus-keygen
|
||||
go build -o lotus-keygen ./cmd/lotus-keygen
|
||||
.PHONY: lotus-keygen
|
||||
BINS+=lotus-keygen
|
||||
|
||||
testground:
|
||||
$(GOCC) build -tags testground -o /dev/null ./cmd/lotus
|
||||
go build -tags testground -o /dev/null ./cmd/lotus
|
||||
.PHONY: testground
|
||||
BINS+=testground
|
||||
|
||||
|
||||
tvx:
|
||||
rm -f tvx
|
||||
$(GOCC) build -o tvx ./cmd/tvx
|
||||
go build -o tvx ./cmd/tvx
|
||||
.PHONY: tvx
|
||||
BINS+=tvx
|
||||
|
||||
install-chainwatch: lotus-chainwatch
|
||||
install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
|
||||
|
||||
lotus-sim: $(BUILD_DEPS)
|
||||
rm -f lotus-sim
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
|
||||
go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
|
||||
.PHONY: lotus-sim
|
||||
BINS+=lotus-sim
|
||||
|
||||
@@ -242,13 +258,21 @@ install-miner-service: install-miner install-daemon-service
|
||||
@echo
|
||||
@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
|
||||
|
||||
install-chainwatch-service: install-chainwatch install-daemon-service
|
||||
mkdir -p /etc/systemd/system
|
||||
mkdir -p /var/log/lotus
|
||||
install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
|
||||
systemctl daemon-reload
|
||||
@echo
|
||||
@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
|
||||
|
||||
install-main-services: install-miner-service
|
||||
|
||||
install-all-services: install-main-services
|
||||
install-all-services: install-main-services install-chainwatch-service
|
||||
|
||||
install-services: install-main-services
|
||||
|
||||
clean-daemon-service: clean-miner-service
|
||||
clean-daemon-service: clean-miner-service clean-chainwatch-service
|
||||
-systemctl stop lotus-daemon
|
||||
-systemctl disable lotus-daemon
|
||||
rm -f /etc/systemd/system/lotus-daemon.service
|
||||
@@ -260,6 +284,12 @@ clean-miner-service:
|
||||
rm -f /etc/systemd/system/lotus-miner.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-chainwatch-service:
|
||||
-systemctl stop lotus-chainwatch
|
||||
-systemctl disable lotus-chainwatch
|
||||
rm -f /etc/systemd/system/lotus-chainwatch.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-main-services: clean-daemon-service
|
||||
|
||||
clean-all-services: clean-main-services
|
||||
@@ -286,26 +316,23 @@ dist-clean:
|
||||
.PHONY: dist-clean
|
||||
|
||||
type-gen: api-gen
|
||||
$(GOCC) run ./gen/main.go
|
||||
$(GOCC) generate -x ./...
|
||||
go run ./gen/main.go
|
||||
go generate -x ./...
|
||||
goimports -w api/
|
||||
|
||||
method-gen: api-gen
|
||||
(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
|
||||
(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
|
||||
|
||||
actors-gen:
|
||||
$(GOCC) run ./chain/actors/agen
|
||||
$(GOCC) fmt ./...
|
||||
go run ./chain/actors/agen
|
||||
go fmt ./...
|
||||
|
||||
api-gen:
|
||||
$(GOCC) run ./gen/api
|
||||
go run ./gen/api
|
||||
goimports -w api
|
||||
goimports -w api
|
||||
.PHONY: api-gen
|
||||
|
||||
cfgdoc-gen:
|
||||
$(GOCC) run ./node/config/cfgdocgen > ./node/config/doc_gen.go
|
||||
|
||||
appimage: lotus
|
||||
rm -rf appimage-builder-cache || true
|
||||
rm AppDir/io.filecoin.lotus.desktop || true
|
||||
@@ -318,9 +345,9 @@ appimage: lotus
|
||||
docsgen: docsgen-md docsgen-openrpc
|
||||
|
||||
docsgen-md-bin: api-gen actors-gen
|
||||
$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
|
||||
go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
|
||||
docsgen-openrpc-bin: api-gen actors-gen
|
||||
$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
|
||||
go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
|
||||
|
||||
docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
|
||||
|
||||
@@ -343,7 +370,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin
|
||||
|
||||
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
|
||||
|
||||
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
|
||||
gen: actors-gen type-gen method-gen docsgen api-gen circleci
|
||||
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
|
||||
.PHONY: gen
|
||||
|
||||
|
||||
+47
-11
@@ -4,12 +4,15 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/journal/alerting"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
metrics "github.com/libp2p/go-libp2p-core/metrics"
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
protocol "github.com/libp2p/go-libp2p-core/protocol"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -24,27 +27,55 @@ import (
|
||||
// * Generate openrpc blobs
|
||||
|
||||
type Common interface {
|
||||
|
||||
// MethodGroup: Auth
|
||||
|
||||
AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
|
||||
AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) //perm:admin
|
||||
|
||||
// MethodGroup: Log
|
||||
// MethodGroup: Net
|
||||
|
||||
LogList(context.Context) ([]string, error) //perm:write
|
||||
LogSetLevel(context.Context, string, string) error //perm:write
|
||||
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
|
||||
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
|
||||
NetConnect(context.Context, peer.AddrInfo) error //perm:write
|
||||
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
|
||||
NetDisconnect(context.Context, peer.ID) error //perm:write
|
||||
NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) //perm:read
|
||||
NetPubsubScores(context.Context) ([]PubsubScore, error) //perm:read
|
||||
NetAutoNatStatus(context.Context) (NatInfo, error) //perm:read
|
||||
NetAgentVersion(ctx context.Context, p peer.ID) (string, error) //perm:read
|
||||
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error) //perm:read
|
||||
|
||||
// LogAlerts returns list of all, active and inactive alerts tracked by the
|
||||
// node
|
||||
LogAlerts(ctx context.Context) ([]alerting.Alert, error) //perm:admin
|
||||
// NetBandwidthStats returns statistics about the nodes total bandwidth
|
||||
// usage and current rate across all peers and protocols.
|
||||
NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
|
||||
|
||||
// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
|
||||
// usage and current rate per peer
|
||||
NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
|
||||
|
||||
// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
|
||||
// usage and current rate per protocol
|
||||
NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
|
||||
|
||||
// ConnectionGater API
|
||||
NetBlockAdd(ctx context.Context, acl NetBlockList) error //perm:admin
|
||||
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
|
||||
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
|
||||
|
||||
// MethodGroup: Common
|
||||
|
||||
// Discover returns an OpenRPC document describing an RPC API.
|
||||
Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
|
||||
|
||||
// ID returns peerID of libp2p node backing this API
|
||||
ID(context.Context) (peer.ID, error) //perm:read
|
||||
|
||||
// Version provides information about API provider
|
||||
Version(context.Context) (APIVersion, error) //perm:read
|
||||
|
||||
// Discover returns an OpenRPC document describing an RPC API.
|
||||
Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
|
||||
LogList(context.Context) ([]string, error) //perm:write
|
||||
LogSetLevel(context.Context, string, string) error //perm:write
|
||||
|
||||
// trigger graceful shutdown
|
||||
Shutdown(context.Context) error //perm:admin
|
||||
@@ -74,3 +105,8 @@ type APIVersion struct {
|
||||
func (v APIVersion) String() string {
|
||||
return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
|
||||
}
|
||||
|
||||
type NatInfo struct {
|
||||
Reachability network.Reachability
|
||||
PublicAddr string
|
||||
}
|
||||
|
||||
+11
-38
@@ -6,20 +6,20 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
|
||||
@@ -59,7 +58,6 @@ const LookbackNoLimit = abi.ChainEpoch(-1)
|
||||
// FullNode API is a low-level interface to the Filecoin network full node
|
||||
type FullNode interface {
|
||||
Common
|
||||
Net
|
||||
|
||||
// MethodGroup: Chain
|
||||
// The Chain method group contains methods for interacting with the
|
||||
@@ -114,11 +112,6 @@ type FullNode interface {
|
||||
// will be returned.
|
||||
ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
|
||||
|
||||
// ChainGetTipSetAfterHeight looks back for a tipset at the specified epoch.
|
||||
// If there are no blocks at the specified epoch, the first non-nil tipset at a later epoch
|
||||
// will be returned.
|
||||
ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
|
||||
|
||||
// ChainReadObj reads ipld nodes referenced by the specified CID from chain
|
||||
// blockstore and returns raw bytes.
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error) //perm:read
|
||||
@@ -170,13 +163,6 @@ type FullNode interface {
|
||||
// If oldmsgskip is set, messages from before the requested roots are also not included.
|
||||
ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
|
||||
|
||||
// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
|
||||
// if supported by the underlying implementation.
|
||||
ChainCheckBlockstore(context.Context) error //perm:admin
|
||||
|
||||
// ChainBlockstoreInfo returns some basic information about the blockstore
|
||||
ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
|
||||
|
||||
// MethodGroup: Beacon
|
||||
// The Beacon method group contains methods for interacting with the random beacon (DRAND)
|
||||
|
||||
@@ -337,7 +323,7 @@ type FullNode interface {
|
||||
// ClientImport imports file under the specified path into filestore.
|
||||
ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin
|
||||
// ClientRemoveImport removes file import
|
||||
ClientRemoveImport(ctx context.Context, importID imports.ID) error //perm:admin
|
||||
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
|
||||
// ClientStartDeal proposes a deal with a miner.
|
||||
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin
|
||||
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
|
||||
@@ -434,8 +420,6 @@ type FullNode interface {
|
||||
StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) //perm:read
|
||||
// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
|
||||
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) //perm:read
|
||||
// StateEncodeParams attempts to encode the provided json params to the binary from
|
||||
StateEncodeParams(ctx context.Context, toActCode cid.Cid, method abi.MethodNum, params json.RawMessage) ([]byte, error) //perm:read
|
||||
|
||||
// StateNetworkName returns the name of the network the node is synced to
|
||||
StateNetworkName(context.Context) (dtypes.NetworkName, error) //perm:read
|
||||
@@ -731,28 +715,17 @@ type MinerSectors struct {
|
||||
|
||||
type ImportRes struct {
|
||||
Root cid.Cid
|
||||
ImportID imports.ID
|
||||
ImportID importmgr.ImportID
|
||||
}
|
||||
|
||||
type Import struct {
|
||||
Key imports.ID
|
||||
Key importmgr.ImportID
|
||||
Err string
|
||||
|
||||
Root *cid.Cid
|
||||
|
||||
// Source is the provenance of the import, e.g. "import", "unknown", else.
|
||||
// Currently useless but may be used in the future.
|
||||
Source string
|
||||
|
||||
// FilePath is the path of the original file. It is important that the file
|
||||
// is retained at this path, because it will be referenced during
|
||||
// the transfer (when we do the UnixFS chunking, we don't duplicate the
|
||||
// leaves, but rather point to chunks of the original data through
|
||||
// positional references).
|
||||
FilePath string
|
||||
|
||||
// CARPath is the path of the CAR file containing the DAG for this import.
|
||||
CARPath string
|
||||
Root *cid.Cid
|
||||
Source string
|
||||
FilePath string
|
||||
CARv2FilePath string
|
||||
}
|
||||
|
||||
type DealInfo struct {
|
||||
@@ -935,7 +908,7 @@ type RetrievalOrder struct {
|
||||
Piece *cid.Cid
|
||||
Size uint64
|
||||
|
||||
FromLocalCAR string // if specified, get data from a local CARv2 file.
|
||||
LocalCARV2FilePath string // if specified, get data from a local CARv2 file.
|
||||
// TODO: support offset
|
||||
Total types.BigInt
|
||||
UnsealPrice types.BigInt
|
||||
|
||||
@@ -33,10 +33,8 @@ type Gateway interface {
|
||||
ChainHead(ctx context.Context) (*types.TipSet, error)
|
||||
ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
|
||||
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
|
||||
ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error)
|
||||
ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainNotify(context.Context) (<-chan []*HeadChange, error)
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
|
||||
@@ -47,7 +45,6 @@ type Gateway interface {
|
||||
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
|
||||
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
|
||||
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
|
||||
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
|
||||
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
metrics "github.com/libp2p/go-libp2p-core/metrics"
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/protocol"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
//
|
||||
// When adding / changing methods in this file:
|
||||
// * Do the change here
|
||||
// * Adjust implementation in `node/impl/`
|
||||
// * Run `make gen` - this will:
|
||||
// * Generate proxy structs
|
||||
// * Generate mocks
|
||||
// * Generate markdown docs
|
||||
// * Generate openrpc blobs
|
||||
|
||||
type Net interface {
|
||||
// MethodGroup: Net
|
||||
|
||||
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
|
||||
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
|
||||
NetConnect(context.Context, peer.AddrInfo) error //perm:write
|
||||
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
|
||||
NetDisconnect(context.Context, peer.ID) error //perm:write
|
||||
NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) //perm:read
|
||||
NetPubsubScores(context.Context) ([]PubsubScore, error) //perm:read
|
||||
NetAutoNatStatus(context.Context) (NatInfo, error) //perm:read
|
||||
NetAgentVersion(ctx context.Context, p peer.ID) (string, error) //perm:read
|
||||
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error) //perm:read
|
||||
|
||||
// NetBandwidthStats returns statistics about the nodes total bandwidth
|
||||
// usage and current rate across all peers and protocols.
|
||||
NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
|
||||
|
||||
// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
|
||||
// usage and current rate per peer
|
||||
NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
|
||||
|
||||
// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
|
||||
// usage and current rate per protocol
|
||||
NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
|
||||
|
||||
// ConnectionGater API
|
||||
NetBlockAdd(ctx context.Context, acl NetBlockList) error //perm:admin
|
||||
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
|
||||
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
|
||||
|
||||
// ID returns peerID of libp2p node backing this API
|
||||
ID(context.Context) (peer.ID, error) //perm:read
|
||||
}
|
||||
|
||||
type CommonNet interface {
|
||||
Common
|
||||
Net
|
||||
}
|
||||
|
||||
type NatInfo struct {
|
||||
Reachability network.Reachability
|
||||
PublicAddr string
|
||||
}
|
||||
+3
-88
@@ -13,13 +13,12 @@ import (
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
"github.com/filecoin-project/go-fil-markets/piecestore"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
|
||||
@@ -42,7 +41,6 @@ import (
|
||||
// StorageMiner is a low-level interface to the Filecoin network storage miner node
|
||||
type StorageMiner interface {
|
||||
Common
|
||||
Net
|
||||
|
||||
ActorAddress(context.Context) (address.Address, error) //perm:read
|
||||
|
||||
@@ -167,52 +165,6 @@ type StorageMiner interface {
|
||||
MarketPendingDeals(ctx context.Context) (PendingDealInfo, error) //perm:write
|
||||
MarketPublishPendingDeals(ctx context.Context) error //perm:admin
|
||||
|
||||
// DagstoreListShards returns information about all shards known to the
|
||||
// DAG store. Only available on nodes running the markets subsystem.
|
||||
DagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:read
|
||||
|
||||
// DagstoreInitializeShard initializes an uninitialized shard.
|
||||
//
|
||||
// Initialization consists of fetching the shard's data (deal payload) from
|
||||
// the storage subsystem, generating an index, and persisting the index
|
||||
// to facilitate later retrievals, and/or to publish to external sources.
|
||||
//
|
||||
// This operation is intended to complement the initial migration. The
|
||||
// migration registers a shard for every unique piece CID, with lazy
|
||||
// initialization. Thus, shards are not initialized immediately to avoid
|
||||
// IO activity competing with proving. Instead, shard are initialized
|
||||
// when first accessed. This method forces the initialization of a shard by
|
||||
// accessing it and immediately releasing it. This is useful to warm up the
|
||||
// cache to facilitate subsequent retrievals, and to generate the indexes
|
||||
// to publish them externally.
|
||||
//
|
||||
// This operation fails if the shard is not in ShardStateNew state.
|
||||
// It blocks until initialization finishes.
|
||||
DagstoreInitializeShard(ctx context.Context, key string) error //perm:write
|
||||
|
||||
// DagstoreRecoverShard attempts to recover a failed shard.
|
||||
//
|
||||
// This operation fails if the shard is not in ShardStateErrored state.
|
||||
// It blocks until recovery finishes. If recovery failed, it returns the
|
||||
// error.
|
||||
DagstoreRecoverShard(ctx context.Context, key string) error //perm:write
|
||||
|
||||
// DagstoreInitializeAll initializes all uninitialized shards in bulk,
|
||||
// according to the policy passed in the parameters.
|
||||
//
|
||||
// It is recommended to set a maximum concurrency to avoid extreme
|
||||
// IO pressure if the storage subsystem has a large amount of deals.
|
||||
//
|
||||
// It returns a stream of events to report progress.
|
||||
DagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:write
|
||||
|
||||
// DagstoreGC runs garbage collection on the DAG store.
|
||||
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
|
||||
|
||||
// RuntimeSubsystems returns the subsystems that are enabled
|
||||
// in this instance.
|
||||
RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
|
||||
|
||||
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
|
||||
DealsList(ctx context.Context) ([]MarketDeal, error) //perm:admin
|
||||
DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin
|
||||
@@ -267,11 +219,6 @@ type SectorLog struct {
|
||||
Message string
|
||||
}
|
||||
|
||||
type SectorPiece struct {
|
||||
Piece abi.PieceInfo
|
||||
DealInfo *PieceDealInfo // nil for pieces which do not appear in deals (e.g. filler pieces)
|
||||
}
|
||||
|
||||
type SectorInfo struct {
|
||||
SectorID abi.SectorNumber
|
||||
State SectorState
|
||||
@@ -279,7 +226,6 @@ type SectorInfo struct {
|
||||
CommR *cid.Cid
|
||||
Proof []byte
|
||||
Deals []abi.DealID
|
||||
Pieces []SectorPiece
|
||||
Ticket SealTicket
|
||||
Seed SealSeed
|
||||
PreCommitMsg *cid.Cid
|
||||
@@ -385,34 +331,3 @@ type DealSchedule struct {
|
||||
StartEpoch abi.ChainEpoch
|
||||
EndEpoch abi.ChainEpoch
|
||||
}
|
||||
|
||||
// DagstoreShardInfo is the serialized form of dagstore.DagstoreShardInfo that
|
||||
// we expose through JSON-RPC to avoid clients having to depend on the
|
||||
// dagstore lib.
|
||||
type DagstoreShardInfo struct {
|
||||
Key string
|
||||
State string
|
||||
Error string
|
||||
}
|
||||
|
||||
// DagstoreShardResult enumerates results per shard.
|
||||
type DagstoreShardResult struct {
|
||||
Key string
|
||||
Success bool
|
||||
Error string
|
||||
}
|
||||
|
||||
type DagstoreInitializeAllParams struct {
|
||||
MaxConcurrency int
|
||||
IncludeSealed bool
|
||||
}
|
||||
|
||||
// DagstoreInitializeAllEvent represents an initialization event.
|
||||
type DagstoreInitializeAllEvent struct {
|
||||
Key string
|
||||
Event string // "start", "end"
|
||||
Success bool
|
||||
Error string
|
||||
Total int
|
||||
Current int
|
||||
}
|
||||
|
||||
+1
-1
@@ -76,7 +76,7 @@ func TestReturnTypes(t *testing.T) {
|
||||
seen[typ] = struct{}{}
|
||||
|
||||
if typ.Kind() == reflect.Interface && typ != bareIface && !typ.Implements(jmarsh) {
|
||||
t.Error("methods can't return interfaces or struct types not implementing json.Marshaller", m.Name)
|
||||
t.Error("methods can't return interfaces", m.Name)
|
||||
}
|
||||
|
||||
switch typ.Kind() {
|
||||
|
||||
@@ -5,7 +5,6 @@ package api
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
abi "github.com/filecoin-project/go-state-types/abi"
|
||||
@@ -18,7 +17,6 @@ import (
|
||||
|
||||
var _ = xerrors.Errorf
|
||||
var _ = cid.Undef
|
||||
var _ = math.E
|
||||
var _ = sort.Sort
|
||||
|
||||
func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
|
||||
|
||||
+34
-12
@@ -16,10 +16,14 @@ import (
|
||||
)
|
||||
|
||||
// NewCommonRPCV0 creates a new http jsonrpc client.
|
||||
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
|
||||
var res v0api.CommonNetStruct
|
||||
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
|
||||
var res v0api.CommonStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
[]interface{}{
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -27,9 +31,11 @@ func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header)
|
||||
// NewFullNodeRPCV0 creates a new http jsonrpc client.
|
||||
func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
var res v0api.FullNodeStruct
|
||||
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
[]interface{}{
|
||||
&res.CommonStruct.Internal,
|
||||
&res.Internal,
|
||||
}, requestHeader)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -38,7 +44,10 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
|
||||
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
var res v1api.FullNodeStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
[]interface{}{
|
||||
&res.CommonStruct.Internal,
|
||||
&res.Internal,
|
||||
}, requestHeader)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -69,10 +78,15 @@ func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.H
|
||||
|
||||
var res v0api.StorageMinerStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader,
|
||||
[]interface{}{
|
||||
&res.CommonStruct.Internal,
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
append([]jsonrpc.Option{
|
||||
rpcenc.ReaderParamEncoder(pushUrl),
|
||||
}, opts...)...)
|
||||
}, opts...)...,
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -85,7 +99,9 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
|
||||
|
||||
var res api.WorkerStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
[]interface{}{
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
rpcenc.ReaderParamEncoder(pushUrl),
|
||||
jsonrpc.WithNoReconnect(),
|
||||
@@ -99,7 +115,9 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
|
||||
func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.Gateway, jsonrpc.ClientCloser, error) {
|
||||
var res api.GatewayStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
[]interface{}{
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
opts...,
|
||||
)
|
||||
@@ -111,7 +129,9 @@ func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header
|
||||
func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.Gateway, jsonrpc.ClientCloser, error) {
|
||||
var res v0api.GatewayStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
[]interface{}{
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
opts...,
|
||||
)
|
||||
@@ -122,7 +142,9 @@ func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header
|
||||
func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Wallet, jsonrpc.ClientCloser, error) {
|
||||
var res api.WalletStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
[]interface{}{
|
||||
&res.Internal,
|
||||
},
|
||||
requestHeader,
|
||||
)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ func main() {
|
||||
|
||||
doc := docgen_openrpc.NewLotusOpenRPCDocument(Comments, GroupDocs)
|
||||
|
||||
i, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
|
||||
i, _, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
|
||||
doc.RegisterReceiverName("Filecoin", i)
|
||||
|
||||
out, err := doc.Discover()
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -16,7 +15,7 @@ func main() {
|
||||
|
||||
groups := make(map[string]*docgen.MethodGroup)
|
||||
|
||||
_, t, permStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
|
||||
_, t, permStruct, commonPermStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
|
||||
|
||||
for i := 0; i < t.NumMethod(); i++ {
|
||||
m := t.Method(i)
|
||||
@@ -89,16 +88,12 @@ func main() {
|
||||
fmt.Printf("### %s\n", m.Name)
|
||||
fmt.Printf("%s\n\n", m.Comment)
|
||||
|
||||
var meth reflect.StructField
|
||||
var ok bool
|
||||
for _, ps := range permStruct {
|
||||
meth, ok = ps.FieldByName(m.Name)
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
meth, ok := permStruct.FieldByName(m.Name)
|
||||
if !ok {
|
||||
panic("no perms for method: " + m.Name)
|
||||
meth, ok = commonPermStruct.FieldByName(m.Name)
|
||||
if !ok {
|
||||
panic("no perms for method: " + m.Name)
|
||||
}
|
||||
}
|
||||
|
||||
perms := meth.Tag.Get("perm")
|
||||
|
||||
+22
-37
@@ -13,13 +13,15 @@ import (
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
"github.com/filecoin-project/go-multistore"
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/google/uuid"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-filestore"
|
||||
"github.com/libp2p/go-libp2p-core/metrics"
|
||||
metrics "github.com/libp2p/go-libp2p-core/metrics"
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/protocol"
|
||||
protocol "github.com/libp2p/go-libp2p-core/protocol"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
|
||||
@@ -42,16 +44,14 @@ import (
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
)
|
||||
|
||||
var ExampleValues = map[reflect.Type]interface{}{
|
||||
reflect.TypeOf(api.MinerSubsystem(0)): api.MinerSubsystem(1),
|
||||
reflect.TypeOf(auth.Permission("")): auth.Permission("write"),
|
||||
reflect.TypeOf(""): "string value",
|
||||
reflect.TypeOf(uint64(42)): uint64(42),
|
||||
reflect.TypeOf(byte(7)): byte(7),
|
||||
reflect.TypeOf([]byte{}): []byte("byte array"),
|
||||
reflect.TypeOf(auth.Permission("")): auth.Permission("write"),
|
||||
reflect.TypeOf(""): "string value",
|
||||
reflect.TypeOf(uint64(42)): uint64(42),
|
||||
reflect.TypeOf(byte(7)): byte(7),
|
||||
reflect.TypeOf([]byte{}): []byte("byte array"),
|
||||
}
|
||||
|
||||
func addExample(v interface{}) {
|
||||
@@ -89,7 +89,8 @@ func init() {
|
||||
addExample(pid)
|
||||
addExample(&pid)
|
||||
|
||||
storeIDExample := imports.ID(50)
|
||||
multistoreIDExample := multistore.StoreID(50)
|
||||
storeIDExample := importmgr.ImportID(50)
|
||||
|
||||
addExample(bitfield.NewFromSet([]uint64{5}))
|
||||
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
|
||||
@@ -122,6 +123,8 @@ func init() {
|
||||
addExample(datatransfer.Ongoing)
|
||||
addExample(storeIDExample)
|
||||
addExample(&storeIDExample)
|
||||
addExample(multistoreIDExample)
|
||||
addExample(&multistoreIDExample)
|
||||
addExample(retrievalmarket.ClientEventDealAccepted)
|
||||
addExample(retrievalmarket.DealStatusNew)
|
||||
addExample(network.ReachabilityPublic)
|
||||
@@ -265,44 +268,27 @@ func init() {
|
||||
|
||||
addExample(api.CheckStatusCode(0))
|
||||
addExample(map[string]interface{}{"abc": 123})
|
||||
addExample(api.MinerSubsystems{
|
||||
api.SubsystemMining,
|
||||
api.SubsystemSealing,
|
||||
api.SubsystemSectorStorage,
|
||||
api.SubsystemMarkets,
|
||||
})
|
||||
addExample(api.DagstoreShardResult{
|
||||
Key: "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
|
||||
Error: "<error>",
|
||||
})
|
||||
addExample(api.DagstoreShardInfo{
|
||||
Key: "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
|
||||
State: "ShardStateAvailable",
|
||||
Error: "<error>",
|
||||
})
|
||||
}
|
||||
|
||||
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
|
||||
|
||||
func GetAPIType(name, pkg string) (i interface{}, t, permStruct, commonPermStruct reflect.Type) {
|
||||
switch pkg {
|
||||
case "api": // latest
|
||||
switch name {
|
||||
case "FullNode":
|
||||
i = &api.FullNodeStruct{}
|
||||
t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.FullNodeStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
|
||||
permStruct = reflect.TypeOf(api.FullNodeStruct{}.Internal)
|
||||
commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
|
||||
case "StorageMiner":
|
||||
i = &api.StorageMinerStruct{}
|
||||
t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.StorageMinerStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
|
||||
permStruct = reflect.TypeOf(api.StorageMinerStruct{}.Internal)
|
||||
commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
|
||||
case "Worker":
|
||||
i = &api.WorkerStruct{}
|
||||
t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
|
||||
permStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
|
||||
commonPermStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
|
||||
default:
|
||||
panic("unknown type")
|
||||
}
|
||||
@@ -311,9 +297,8 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
|
||||
case "FullNode":
|
||||
i = v0api.FullNodeStruct{}
|
||||
t = reflect.TypeOf(new(struct{ v0api.FullNode })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(v0api.FullNodeStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(v0api.CommonStruct{}.Internal))
|
||||
permStruct = append(permStruct, reflect.TypeOf(v0api.NetStruct{}.Internal))
|
||||
permStruct = reflect.TypeOf(v0api.FullNodeStruct{}.Internal)
|
||||
commonPermStruct = reflect.TypeOf(v0api.CommonStruct{}.Internal)
|
||||
default:
|
||||
panic("unknown type")
|
||||
}
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// MinerSubsystem represents a miner subsystem. Int and string values are not
|
||||
// guaranteed to be stable over time is not
|
||||
// guaranteed to be stable over time.
|
||||
type MinerSubsystem int
|
||||
|
||||
const (
|
||||
// SubsystemUnknown is a placeholder for the zero value. It should never
|
||||
// be used.
|
||||
SubsystemUnknown MinerSubsystem = iota
|
||||
// SubsystemMarkets signifies the storage and retrieval
|
||||
// deal-making subsystem.
|
||||
SubsystemMarkets
|
||||
// SubsystemMining signifies the mining subsystem.
|
||||
SubsystemMining
|
||||
// SubsystemSealing signifies the sealing subsystem.
|
||||
SubsystemSealing
|
||||
// SubsystemSectorStorage signifies the sector storage subsystem.
|
||||
SubsystemSectorStorage
|
||||
)
|
||||
|
||||
var MinerSubsystemToString = map[MinerSubsystem]string{
|
||||
SubsystemUnknown: "Unknown",
|
||||
SubsystemMarkets: "Markets",
|
||||
SubsystemMining: "Mining",
|
||||
SubsystemSealing: "Sealing",
|
||||
SubsystemSectorStorage: "SectorStorage",
|
||||
}
|
||||
|
||||
var MinerSubsystemToID = map[string]MinerSubsystem{
|
||||
"Unknown": SubsystemUnknown,
|
||||
"Markets": SubsystemMarkets,
|
||||
"Mining": SubsystemMining,
|
||||
"Sealing": SubsystemSealing,
|
||||
"SectorStorage": SubsystemSectorStorage,
|
||||
}
|
||||
|
||||
func (ms MinerSubsystem) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(MinerSubsystemToString[ms])
|
||||
}
|
||||
|
||||
func (ms *MinerSubsystem) UnmarshalJSON(b []byte) error {
|
||||
var j string
|
||||
err := json.Unmarshal(b, &j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, ok := MinerSubsystemToID[j]
|
||||
if !ok {
|
||||
*ms = SubsystemUnknown
|
||||
} else {
|
||||
*ms = s
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type MinerSubsystems []MinerSubsystem
|
||||
|
||||
func (ms MinerSubsystems) Has(entry MinerSubsystem) bool {
|
||||
for _, v := range ms {
|
||||
if v == entry {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (ms MinerSubsystem) String() string {
|
||||
s, ok := MinerSubsystemToString[ms]
|
||||
if !ok {
|
||||
return MinerSubsystemToString[SubsystemUnknown]
|
||||
}
|
||||
return s
|
||||
}
|
||||
+2
-78
@@ -6,7 +6,6 @@ package mocks
|
||||
|
||||
import (
|
||||
context "context"
|
||||
json "encoding/json"
|
||||
reflect "reflect"
|
||||
|
||||
address "github.com/filecoin-project/go-address"
|
||||
@@ -24,10 +23,9 @@ import (
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
alerting "github.com/filecoin-project/lotus/journal/alerting"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
imports "github.com/filecoin-project/lotus/node/repo/imports"
|
||||
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
|
||||
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
@@ -107,35 +105,6 @@ func (mr *MockFullNodeMockRecorder) BeaconGetEntry(arg0, arg1 interface{}) *gomo
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BeaconGetEntry", reflect.TypeOf((*MockFullNode)(nil).BeaconGetEntry), arg0, arg1)
|
||||
}
|
||||
|
||||
// ChainBlockstoreInfo mocks base method.
|
||||
func (m *MockFullNode) ChainBlockstoreInfo(arg0 context.Context) (map[string]interface{}, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ChainBlockstoreInfo", arg0)
|
||||
ret0, _ := ret[0].(map[string]interface{})
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// ChainBlockstoreInfo indicates an expected call of ChainBlockstoreInfo.
|
||||
func (mr *MockFullNodeMockRecorder) ChainBlockstoreInfo(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainBlockstoreInfo", reflect.TypeOf((*MockFullNode)(nil).ChainBlockstoreInfo), arg0)
|
||||
}
|
||||
|
||||
// ChainCheckBlockstore mocks base method.
|
||||
func (m *MockFullNode) ChainCheckBlockstore(arg0 context.Context) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ChainCheckBlockstore", arg0)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// ChainCheckBlockstore indicates an expected call of ChainCheckBlockstore.
|
||||
func (mr *MockFullNodeMockRecorder) ChainCheckBlockstore(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainCheckBlockstore", reflect.TypeOf((*MockFullNode)(nil).ChainCheckBlockstore), arg0)
|
||||
}
|
||||
|
||||
// ChainDeleteObj mocks base method.
|
||||
func (m *MockFullNode) ChainDeleteObj(arg0 context.Context, arg1 cid.Cid) error {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -345,21 +314,6 @@ func (mr *MockFullNodeMockRecorder) ChainGetTipSet(arg0, arg1 interface{}) *gomo
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetTipSet", reflect.TypeOf((*MockFullNode)(nil).ChainGetTipSet), arg0, arg1)
|
||||
}
|
||||
|
||||
// ChainGetTipSetAfterHeight mocks base method.
|
||||
func (m *MockFullNode) ChainGetTipSetAfterHeight(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) (*types.TipSet, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ChainGetTipSetAfterHeight", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(*types.TipSet)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// ChainGetTipSetAfterHeight indicates an expected call of ChainGetTipSetAfterHeight.
|
||||
func (mr *MockFullNodeMockRecorder) ChainGetTipSetAfterHeight(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetTipSetAfterHeight", reflect.TypeOf((*MockFullNode)(nil).ChainGetTipSetAfterHeight), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// ChainGetTipSetByHeight mocks base method.
|
||||
func (m *MockFullNode) ChainGetTipSetByHeight(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) (*types.TipSet, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -777,7 +731,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
|
||||
}
|
||||
|
||||
// ClientRemoveImport mocks base method.
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 imports.ID) error {
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
@@ -996,21 +950,6 @@ func (mr *MockFullNodeMockRecorder) ID(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockFullNode)(nil).ID), arg0)
|
||||
}
|
||||
|
||||
// LogAlerts mocks base method.
|
||||
func (m *MockFullNode) LogAlerts(arg0 context.Context) ([]alerting.Alert, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "LogAlerts", arg0)
|
||||
ret0, _ := ret[0].([]alerting.Alert)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// LogAlerts indicates an expected call of LogAlerts.
|
||||
func (mr *MockFullNodeMockRecorder) LogAlerts(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogAlerts", reflect.TypeOf((*MockFullNode)(nil).LogAlerts), arg0)
|
||||
}
|
||||
|
||||
// LogList mocks base method.
|
||||
func (m *MockFullNode) LogList(arg0 context.Context) ([]string, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2261,21 +2200,6 @@ func (mr *MockFullNodeMockRecorder) StateDecodeParams(arg0, arg1, arg2, arg3, ar
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateDecodeParams", reflect.TypeOf((*MockFullNode)(nil).StateDecodeParams), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// StateEncodeParams mocks base method.
|
||||
func (m *MockFullNode) StateEncodeParams(arg0 context.Context, arg1 cid.Cid, arg2 abi.MethodNum, arg3 json.RawMessage) ([]byte, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateEncodeParams", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].([]byte)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateEncodeParams indicates an expected call of StateEncodeParams.
|
||||
func (mr *MockFullNodeMockRecorder) StateEncodeParams(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateEncodeParams", reflect.TypeOf((*MockFullNode)(nil).StateEncodeParams), arg0, arg1, arg2, arg3)
|
||||
}
|
||||
|
||||
// StateGetActor mocks base method.
|
||||
func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.Actor, error) {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
+6
-11
@@ -16,33 +16,28 @@ const (
|
||||
var AllPermissions = []auth.Permission{PermRead, PermWrite, PermSign, PermAdmin}
|
||||
var DefaultPerms = []auth.Permission{PermRead}
|
||||
|
||||
func permissionedProxies(in, out interface{}) {
|
||||
outs := GetInternalStructs(out)
|
||||
for _, o := range outs {
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, in, o)
|
||||
}
|
||||
}
|
||||
|
||||
func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
|
||||
var out StorageMinerStruct
|
||||
permissionedProxies(a, &out)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
|
||||
return &out
|
||||
}
|
||||
|
||||
func PermissionedFullAPI(a FullNode) FullNode {
|
||||
var out FullNodeStruct
|
||||
permissionedProxies(a, &out)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
|
||||
return &out
|
||||
}
|
||||
|
||||
func PermissionedWorkerAPI(a Worker) Worker {
|
||||
var out WorkerStruct
|
||||
permissionedProxies(a, &out)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
|
||||
return &out
|
||||
}
|
||||
|
||||
func PermissionedWalletAPI(a Wallet) Wallet {
|
||||
var out WalletStruct
|
||||
permissionedProxies(a, &out)
|
||||
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
|
||||
return &out
|
||||
}
|
||||
|
||||
+514
-1815
File diff suppressed because it is too large
Load Diff
@@ -1,30 +0,0 @@
|
||||
package api
|
||||
|
||||
import "reflect"
|
||||
|
||||
var _internalField = "Internal"
|
||||
|
||||
// GetInternalStructs extracts all pointers to 'Internal' sub-structs from the provided pointer to a proxy struct
|
||||
func GetInternalStructs(in interface{}) []interface{} {
|
||||
return getInternalStructs(reflect.ValueOf(in).Elem())
|
||||
}
|
||||
|
||||
func getInternalStructs(rv reflect.Value) []interface{} {
|
||||
var out []interface{}
|
||||
|
||||
internal := rv.FieldByName(_internalField)
|
||||
ii := internal.Addr().Interface()
|
||||
out = append(out, ii)
|
||||
|
||||
for i := 0; i < rv.NumField(); i++ {
|
||||
if rv.Type().Field(i).Name == _internalField {
|
||||
continue
|
||||
}
|
||||
|
||||
sub := getInternalStructs(rv.Field(i))
|
||||
|
||||
out = append(out, sub...)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type StrA struct {
|
||||
StrB
|
||||
|
||||
Internal struct {
|
||||
A int
|
||||
}
|
||||
}
|
||||
|
||||
type StrB struct {
|
||||
Internal struct {
|
||||
B int
|
||||
}
|
||||
}
|
||||
|
||||
type StrC struct {
|
||||
Internal struct {
|
||||
Internal struct {
|
||||
C int
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetInternalStructs(t *testing.T) {
|
||||
var proxy StrA
|
||||
|
||||
sts := GetInternalStructs(&proxy)
|
||||
require.Len(t, sts, 2)
|
||||
|
||||
sa := sts[0].(*struct{ A int })
|
||||
sa.A = 3
|
||||
sb := sts[1].(*struct{ B int })
|
||||
sb.B = 4
|
||||
|
||||
require.Equal(t, 3, proxy.Internal.A)
|
||||
require.Equal(t, 4, proxy.StrB.Internal.B)
|
||||
}
|
||||
|
||||
func TestNestedInternalStructs(t *testing.T) {
|
||||
var proxy StrC
|
||||
|
||||
// check that only the top-level internal struct gets picked up
|
||||
|
||||
sts := GetInternalStructs(&proxy)
|
||||
require.Len(t, sts, 1)
|
||||
|
||||
sa := sts[0].(*struct {
|
||||
Internal struct {
|
||||
C int
|
||||
}
|
||||
})
|
||||
sa.Internal.C = 5
|
||||
|
||||
require.Equal(t, 5, proxy.Internal.Internal.C)
|
||||
}
|
||||
+2
-3
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=v0mocks/mock_full.go -package=v0mocks . FullNode
|
||||
@@ -46,7 +46,6 @@ import (
|
||||
// FullNode API is a low-level interface to the Filecoin network full node
|
||||
type FullNode interface {
|
||||
Common
|
||||
Net
|
||||
|
||||
// MethodGroup: Chain
|
||||
// The Chain method group contains methods for interacting with the
|
||||
@@ -305,7 +304,7 @@ type FullNode interface {
|
||||
// ClientImport imports file under the specified path into filestore.
|
||||
ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
|
||||
// ClientRemoveImport removes file import
|
||||
ClientRemoveImport(ctx context.Context, importID imports.ID) error //perm:admin
|
||||
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
|
||||
// ClientStartDeal proposes a deal with a miner.
|
||||
ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
|
||||
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
|
||||
|
||||
@@ -5,15 +5,8 @@ import (
|
||||
)
|
||||
|
||||
type Common = api.Common
|
||||
type Net = api.Net
|
||||
type CommonNet = api.CommonNet
|
||||
|
||||
type CommonStruct = api.CommonStruct
|
||||
type CommonStub = api.CommonStub
|
||||
type NetStruct = api.NetStruct
|
||||
type NetStub = api.NetStub
|
||||
type CommonNetStruct = api.CommonNetStruct
|
||||
type CommonNetStub = api.CommonNetStub
|
||||
|
||||
type StorageMiner = api.StorageMiner
|
||||
type StorageMinerStruct = api.StorageMinerStruct
|
||||
|
||||
+212
-842
File diff suppressed because it is too large
Load Diff
@@ -23,10 +23,9 @@ import (
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
alerting "github.com/filecoin-project/lotus/journal/alerting"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
imports "github.com/filecoin-project/lotus/node/repo/imports"
|
||||
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
|
||||
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
@@ -732,7 +731,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
|
||||
}
|
||||
|
||||
// ClientRemoveImport mocks base method.
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 imports.ID) error {
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
@@ -951,21 +950,6 @@ func (mr *MockFullNodeMockRecorder) ID(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockFullNode)(nil).ID), arg0)
|
||||
}
|
||||
|
||||
// LogAlerts mocks base method.
|
||||
func (m *MockFullNode) LogAlerts(arg0 context.Context) ([]alerting.Alert, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "LogAlerts", arg0)
|
||||
ret0, _ := ret[0].([]alerting.Alert)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// LogAlerts indicates an expected call of LogAlerts.
|
||||
func (mr *MockFullNodeMockRecorder) LogAlerts(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogAlerts", reflect.TypeOf((*MockFullNode)(nil).LogAlerts), arg0)
|
||||
}
|
||||
|
||||
// LogList mocks base method.
|
||||
func (m *MockFullNode) LogList(arg0 context.Context) ([]string, error) {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
+35
-439
@@ -4,15 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/badger/v2"
|
||||
"github.com/dgraph-io/badger/v2/options"
|
||||
"github.com/dgraph-io/badger/v2/pb"
|
||||
"github.com/multiformats/go-base32"
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -76,45 +72,19 @@ func (b *badgerLogger) Warningf(format string, args ...interface{}) {
|
||||
b.skip2.Warnf(format, args...)
|
||||
}
|
||||
|
||||
// bsState is the current blockstore state
|
||||
type bsState int
|
||||
|
||||
const (
|
||||
// stateOpen signifies an open blockstore
|
||||
stateOpen bsState = iota
|
||||
// stateClosing signifies a blockstore that is currently closing
|
||||
stateOpen = iota
|
||||
stateClosing
|
||||
// stateClosed signifies a blockstore that has been colosed
|
||||
stateClosed
|
||||
)
|
||||
|
||||
type bsMoveState int
|
||||
|
||||
const (
|
||||
// moveStateNone signifies that there is no move in progress
|
||||
moveStateNone bsMoveState = iota
|
||||
// moveStateMoving signifies that there is a move in a progress
|
||||
moveStateMoving
|
||||
// moveStateCleanup signifies that a move has completed or aborted and we are cleaning up
|
||||
moveStateCleanup
|
||||
// moveStateLock signifies that an exclusive lock has been acquired
|
||||
moveStateLock
|
||||
)
|
||||
|
||||
// Blockstore is a badger-backed IPLD blockstore.
|
||||
type Blockstore struct {
|
||||
stateLk sync.RWMutex
|
||||
state bsState
|
||||
state int
|
||||
viewers sync.WaitGroup
|
||||
|
||||
moveMx sync.Mutex
|
||||
moveCond sync.Cond
|
||||
moveState bsMoveState
|
||||
rlock int
|
||||
|
||||
db *badger.DB
|
||||
dbNext *badger.DB // when moving
|
||||
opts Options
|
||||
DB *badger.DB
|
||||
|
||||
prefixing bool
|
||||
prefix []byte
|
||||
@@ -125,7 +95,6 @@ var _ blockstore.Blockstore = (*Blockstore)(nil)
|
||||
var _ blockstore.Viewer = (*Blockstore)(nil)
|
||||
var _ blockstore.BlockstoreIterator = (*Blockstore)(nil)
|
||||
var _ blockstore.BlockstoreGC = (*Blockstore)(nil)
|
||||
var _ blockstore.BlockstoreSize = (*Blockstore)(nil)
|
||||
var _ io.Closer = (*Blockstore)(nil)
|
||||
|
||||
// Open creates a new badger-backed blockstore, with the supplied options.
|
||||
@@ -140,15 +109,13 @@ func Open(opts Options) (*Blockstore, error) {
|
||||
return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
|
||||
}
|
||||
|
||||
bs := &Blockstore{db: db, opts: opts}
|
||||
bs := &Blockstore{DB: db}
|
||||
if p := opts.Prefix; p != "" {
|
||||
bs.prefixing = true
|
||||
bs.prefix = []byte(p)
|
||||
bs.prefixLen = len(bs.prefix)
|
||||
}
|
||||
|
||||
bs.moveCond.L = &bs.moveMx
|
||||
|
||||
return bs, nil
|
||||
}
|
||||
|
||||
@@ -172,7 +139,7 @@ func (b *Blockstore) Close() error {
|
||||
// wait for all accesses to complete
|
||||
b.viewers.Wait()
|
||||
|
||||
return b.db.Close()
|
||||
return b.DB.Close()
|
||||
}
|
||||
|
||||
func (b *Blockstore) access() error {
|
||||
@@ -194,254 +161,12 @@ func (b *Blockstore) isOpen() bool {
|
||||
return b.state == stateOpen
|
||||
}
|
||||
|
||||
// lockDB/unlockDB implement a recursive lock contingent on move state
|
||||
func (b *Blockstore) lockDB() {
|
||||
b.moveMx.Lock()
|
||||
defer b.moveMx.Unlock()
|
||||
|
||||
if b.rlock == 0 {
|
||||
for b.moveState == moveStateLock {
|
||||
b.moveCond.Wait()
|
||||
}
|
||||
// CollectGarbage runs garbage collection on the value log
|
||||
func (b *Blockstore) CollectGarbage() error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.rlock++
|
||||
}
|
||||
|
||||
func (b *Blockstore) unlockDB() {
|
||||
b.moveMx.Lock()
|
||||
defer b.moveMx.Unlock()
|
||||
|
||||
b.rlock--
|
||||
if b.rlock == 0 && b.moveState == moveStateLock {
|
||||
b.moveCond.Broadcast()
|
||||
}
|
||||
}
|
||||
|
||||
// lockMove/unlockMove implement an exclusive lock of move state
|
||||
func (b *Blockstore) lockMove() {
|
||||
b.moveMx.Lock()
|
||||
b.moveState = moveStateLock
|
||||
for b.rlock > 0 {
|
||||
b.moveCond.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Blockstore) unlockMove(state bsMoveState) {
|
||||
b.moveState = state
|
||||
b.moveCond.Broadcast()
|
||||
b.moveMx.Unlock()
|
||||
}
|
||||
|
||||
// movingGC moves the blockstore to a new path, adjacent to the current path, and creates
|
||||
// a symlink from the current path to the new path; the old blockstore is deleted.
|
||||
//
|
||||
// The blockstore MUST accept new writes during the move and ensure that these
|
||||
// are persisted to the new blockstore; if a failure occurs aboring the move,
|
||||
// then they must be peristed to the old blockstore.
|
||||
// In short, the blockstore must not lose data from new writes during the move.
|
||||
func (b *Blockstore) movingGC() error {
|
||||
// this inlines moveLock/moveUnlock for the initial state check to prevent a second move
|
||||
// while one is in progress without clobbering state
|
||||
b.moveMx.Lock()
|
||||
if b.moveState != moveStateNone {
|
||||
b.moveMx.Unlock()
|
||||
return fmt.Errorf("move in progress")
|
||||
}
|
||||
|
||||
b.moveState = moveStateLock
|
||||
for b.rlock > 0 {
|
||||
b.moveCond.Wait()
|
||||
}
|
||||
|
||||
b.moveState = moveStateMoving
|
||||
b.moveCond.Broadcast()
|
||||
b.moveMx.Unlock()
|
||||
|
||||
var newPath string
|
||||
|
||||
defer func() {
|
||||
b.lockMove()
|
||||
|
||||
dbNext := b.dbNext
|
||||
b.dbNext = nil
|
||||
|
||||
var state bsMoveState
|
||||
if dbNext != nil {
|
||||
state = moveStateCleanup
|
||||
} else {
|
||||
state = moveStateNone
|
||||
}
|
||||
|
||||
b.unlockMove(state)
|
||||
|
||||
if dbNext != nil {
|
||||
// the move failed and we have a left-over db; delete it.
|
||||
err := dbNext.Close()
|
||||
if err != nil {
|
||||
log.Warnf("error closing badger db: %s", err)
|
||||
}
|
||||
b.deleteDB(newPath)
|
||||
|
||||
b.lockMove()
|
||||
b.unlockMove(moveStateNone)
|
||||
}
|
||||
}()
|
||||
|
||||
// we resolve symlinks to create the new path in the adjacent to the old path.
|
||||
// this allows the user to symlink the db directory into a separate filesystem.
|
||||
basePath := b.opts.Dir
|
||||
linkPath, err := filepath.EvalSymlinks(basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error resolving symlink %s: %w", basePath, err)
|
||||
}
|
||||
|
||||
if basePath == linkPath {
|
||||
newPath = basePath
|
||||
} else {
|
||||
// we do this dance to create a name adjacent to the current one, while avoiding clown
|
||||
// shoes with multiple moves (i.e. we can't just take the basename of the linkPath, as it
|
||||
// could have been created in a previous move and have the timestamp suffix, which would then
|
||||
// perpetuate itself.
|
||||
name := filepath.Base(basePath)
|
||||
dir := filepath.Dir(linkPath)
|
||||
newPath = filepath.Join(dir, name)
|
||||
}
|
||||
newPath = fmt.Sprintf("%s.%d", newPath, time.Now().UnixNano())
|
||||
|
||||
log.Infof("moving blockstore from %s to %s", b.opts.Dir, newPath)
|
||||
|
||||
opts := b.opts
|
||||
opts.Dir = newPath
|
||||
opts.ValueDir = newPath
|
||||
|
||||
dbNew, err := badger.Open(opts.Options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open badger blockstore in %s: %w", newPath, err)
|
||||
}
|
||||
|
||||
b.lockMove()
|
||||
b.dbNext = dbNew
|
||||
b.unlockMove(moveStateMoving)
|
||||
|
||||
log.Info("copying blockstore")
|
||||
err = b.doCopy(b.db, b.dbNext)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error moving badger blockstore to %s: %w", newPath, err)
|
||||
}
|
||||
|
||||
b.lockMove()
|
||||
dbOld := b.db
|
||||
b.db = b.dbNext
|
||||
b.dbNext = nil
|
||||
b.unlockMove(moveStateCleanup)
|
||||
|
||||
err = dbOld.Close()
|
||||
if err != nil {
|
||||
log.Warnf("error closing old badger db: %s", err)
|
||||
}
|
||||
|
||||
// this is the canonical db path; this is where our db lives.
|
||||
dbPath := b.opts.Dir
|
||||
|
||||
// we first move the existing db out of the way, and only delete it after we have symlinked the
|
||||
// new db to the canonical path
|
||||
backupPath := fmt.Sprintf("%s.old.%d", dbPath, time.Now().Unix())
|
||||
if err = os.Rename(dbPath, backupPath); err != nil {
|
||||
// this is not catastrophic in the sense that we have not lost any data.
|
||||
// but it is pretty bad, as the db path points to the old db, while we are now using to the new
|
||||
// db; we can't continue and leave a ticking bomb for the next restart.
|
||||
// so a panic is appropriate and user can fix.
|
||||
panic(fmt.Errorf("error renaming old badger db dir from %s to %s: %w; USER ACTION REQUIRED", dbPath, backupPath, err)) //nolint
|
||||
}
|
||||
|
||||
if err = symlink(newPath, dbPath); err != nil {
|
||||
// same here; the db path is pointing to the void. panic and let the user fix.
|
||||
panic(fmt.Errorf("error symlinking new badger db dir from %s to %s: %w; USER ACTION REQUIRED", newPath, dbPath, err)) //nolint
|
||||
}
|
||||
|
||||
// the delete follows symlinks
|
||||
b.deleteDB(backupPath)
|
||||
|
||||
log.Info("moving blockstore done")
|
||||
return nil
|
||||
}
|
||||
|
||||
// symlink creates a symlink from path to linkTo; the link is relative if the two are
|
||||
// in the same directory
|
||||
func symlink(path, linkTo string) error {
|
||||
resolvedPathDir, err := filepath.EvalSymlinks(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error resolving links in %s: %w", path, err)
|
||||
}
|
||||
|
||||
resolvedLinkDir, err := filepath.EvalSymlinks(filepath.Dir(linkTo))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error resolving links in %s: %w", linkTo, err)
|
||||
}
|
||||
|
||||
if resolvedPathDir == resolvedLinkDir {
|
||||
path = filepath.Base(path)
|
||||
}
|
||||
|
||||
return os.Symlink(path, linkTo)
|
||||
}
|
||||
|
||||
// doCopy copies a badger blockstore to another, with an optional filter; if the filter
|
||||
// is not nil, then only cids that satisfy the filter will be copied.
|
||||
func (b *Blockstore) doCopy(from, to *badger.DB) error {
|
||||
workers := runtime.NumCPU() / 2
|
||||
if workers < 2 {
|
||||
workers = 2
|
||||
}
|
||||
|
||||
stream := from.NewStream()
|
||||
stream.NumGo = workers
|
||||
stream.LogPrefix = "doCopy"
|
||||
stream.Send = func(list *pb.KVList) error {
|
||||
batch := to.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for _, kv := range list.Kv {
|
||||
if kv.Key == nil || kv.Value == nil {
|
||||
continue
|
||||
}
|
||||
if err := batch.Set(kv.Key, kv.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return batch.Flush()
|
||||
}
|
||||
|
||||
return stream.Orchestrate(context.Background())
|
||||
}
|
||||
|
||||
func (b *Blockstore) deleteDB(path string) {
|
||||
// follow symbolic links, otherwise the data wil be left behind
|
||||
linkPath, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
log.Warnf("error resolving symlinks in %s", path)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("removing data directory %s", linkPath)
|
||||
if err := os.RemoveAll(linkPath); err != nil {
|
||||
log.Warnf("error deleting db at %s: %s", linkPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
if path != linkPath {
|
||||
log.Infof("removing link %s", path)
|
||||
if err := os.Remove(path); err != nil {
|
||||
log.Warnf("error removing symbolic link %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Blockstore) onlineGC() error {
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
defer b.viewers.Done()
|
||||
|
||||
// compact first to gather the necessary statistics for GC
|
||||
nworkers := runtime.NumCPU() / 2
|
||||
@@ -449,13 +174,13 @@ func (b *Blockstore) onlineGC() error {
|
||||
nworkers = 2
|
||||
}
|
||||
|
||||
err := b.db.Flatten(nworkers)
|
||||
err := b.DB.Flatten(nworkers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for err == nil {
|
||||
err = b.db.RunValueLogGC(0.125)
|
||||
err = b.DB.RunValueLogGC(0.125)
|
||||
}
|
||||
|
||||
if err == badger.ErrNoRewrite {
|
||||
@@ -466,63 +191,6 @@ func (b *Blockstore) onlineGC() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// CollectGarbage compacts and runs garbage collection on the value log;
|
||||
// implements the BlockstoreGC trait
|
||||
func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
var options blockstore.BlockstoreGCOptions
|
||||
for _, opt := range opts {
|
||||
err := opt(&options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if options.FullGC {
|
||||
return b.movingGC()
|
||||
}
|
||||
|
||||
return b.onlineGC()
|
||||
}
|
||||
|
||||
// Size returns the aggregate size of the blockstore
|
||||
func (b *Blockstore) Size() (int64, error) {
|
||||
if err := b.access(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
lsm, vlog := b.db.Size()
|
||||
size := lsm + vlog
|
||||
|
||||
if size == 0 {
|
||||
// badger reports a 0 size on symlinked directories... sigh
|
||||
dir := b.opts.Dir
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
path := filepath.Join(dir, e.Name())
|
||||
finfo, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size += finfo.Size()
|
||||
}
|
||||
}
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// View implements blockstore.Viewer, which leverages zero-copy read-only
|
||||
// access to values.
|
||||
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
|
||||
@@ -531,15 +199,12 @@ func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
return b.db.View(func(txn *badger.Txn) error {
|
||||
return b.DB.View(func(txn *badger.Txn) error {
|
||||
switch item, err := txn.Get(k); err {
|
||||
case nil:
|
||||
return item.Value(fn)
|
||||
@@ -558,15 +223,12 @@ func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
err := b.DB.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(k)
|
||||
return err
|
||||
})
|
||||
@@ -592,16 +254,13 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
var val []byte
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
err := b.DB.View(func(txn *badger.Txn) error {
|
||||
switch item, err := txn.Get(k); err {
|
||||
case nil:
|
||||
val, err = item.ValueCopy(nil)
|
||||
@@ -625,16 +284,13 @@ func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
var size int
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
err := b.DB.View(func(txn *badger.Txn) error {
|
||||
switch item, err := txn.Get(k); err {
|
||||
case nil:
|
||||
size = int(item.ValueSize())
|
||||
@@ -658,36 +314,18 @@ func (b *Blockstore) Put(block blocks.Block) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(block.Cid())
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
put := func(db *badger.DB) error {
|
||||
err := db.Update(func(txn *badger.Txn) error {
|
||||
return txn.Set(k, block.RawData())
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put block in badger blockstore: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
err := b.DB.Update(func(txn *badger.Txn) error {
|
||||
return txn.Set(k, block.RawData())
|
||||
})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to put block in badger blockstore: %w", err)
|
||||
}
|
||||
|
||||
if err := put(b.db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if b.dbNext != nil {
|
||||
if err := put(b.dbNext); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// PutMany implements Blockstore.PutMany.
|
||||
@@ -697,9 +335,6 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
// toReturn tracks the byte slices to return to the pool, if we're using key
|
||||
// prefixing. we can't return each slice to the pool after each Set, because
|
||||
// badger holds on to the slice.
|
||||
@@ -713,45 +348,24 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
|
||||
}()
|
||||
}
|
||||
|
||||
keys := make([][]byte, 0, len(blocks))
|
||||
batch := b.DB.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for _, block := range blocks {
|
||||
k, pooled := b.PooledStorageKey(block.Cid())
|
||||
if pooled {
|
||||
toReturn = append(toReturn, k)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
put := func(db *badger.DB) error {
|
||||
batch := db.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for i, block := range blocks {
|
||||
k := keys[i]
|
||||
if err := batch.Set(k, block.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err := batch.Flush()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := put(b.db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if b.dbNext != nil {
|
||||
if err := put(b.dbNext); err != nil {
|
||||
if err := batch.Set(k, block.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
err := batch.Flush()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteBlock implements Blockstore.DeleteBlock.
|
||||
@@ -761,15 +375,12 @@ func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
k, pooled := b.PooledStorageKey(cid)
|
||||
if pooled {
|
||||
defer KeyPool.Put(k)
|
||||
}
|
||||
|
||||
return b.db.Update(func(txn *badger.Txn) error {
|
||||
return b.DB.Update(func(txn *badger.Txn) error {
|
||||
return txn.Delete(k)
|
||||
})
|
||||
}
|
||||
@@ -780,9 +391,6 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
// toReturn tracks the byte slices to return to the pool, if we're using key
|
||||
// prefixing. we can't return each slice to the pool after each Set, because
|
||||
// badger holds on to the slice.
|
||||
@@ -796,7 +404,7 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
|
||||
}()
|
||||
}
|
||||
|
||||
batch := b.db.NewWriteBatch()
|
||||
batch := b.DB.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for _, cid := range cids {
|
||||
@@ -822,10 +430,7 @@ func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
txn := b.db.NewTransaction(false)
|
||||
txn := b.DB.NewTransaction(false)
|
||||
opts := badger.IteratorOptions{PrefetchSize: 100}
|
||||
if b.prefixing {
|
||||
opts.Prefix = b.prefix
|
||||
@@ -879,10 +484,7 @@ func (b *Blockstore) ForEachKey(f func(cid.Cid) error) error {
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
txn := b.db.NewTransaction(false)
|
||||
txn := b.DB.NewTransaction(false)
|
||||
defer txn.Discard()
|
||||
|
||||
opts := badger.IteratorOptions{PrefetchSize: 100}
|
||||
@@ -977,9 +579,3 @@ func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte {
|
||||
}
|
||||
return dst[:reqsize]
|
||||
}
|
||||
|
||||
// this method is added for lotus-shed needs
|
||||
// WARNING: THIS IS COMPLETELY UNSAFE; DONT USE THIS IN PRODUCTION CODE
|
||||
func (b *Blockstore) DB() *badger.DB {
|
||||
return b.db
|
||||
}
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
package badgerbs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
@@ -96,180 +89,3 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
|
||||
return Open(optsSupplier(path))
|
||||
}
|
||||
}
|
||||
|
||||
func testMove(t *testing.T, optsF func(string) Options) {
|
||||
basePath, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(basePath, "db")
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = os.RemoveAll(basePath)
|
||||
})
|
||||
|
||||
db, err := Open(optsF(dbPath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer db.Close() //nolint
|
||||
|
||||
var have []blocks.Block
|
||||
var deleted []cid.Cid
|
||||
|
||||
// add some blocks
|
||||
for i := 0; i < 10; i++ {
|
||||
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
||||
err := db.Put(blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have = append(have, blk)
|
||||
}
|
||||
|
||||
// delete some of them
|
||||
for i := 5; i < 10; i++ {
|
||||
c := have[i].Cid()
|
||||
err := db.DeleteBlock(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleted = append(deleted, c)
|
||||
}
|
||||
have = have[:5]
|
||||
|
||||
// start a move concurrent with some more puts
|
||||
g := new(errgroup.Group)
|
||||
g.Go(func() error {
|
||||
for i := 10; i < 1000; i++ {
|
||||
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
||||
err := db.Put(blk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
have = append(have, blk)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
g.Go(func() error {
|
||||
return db.CollectGarbage(blockstore.WithFullGC(true))
|
||||
})
|
||||
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// now check that we have all the blocks in have and none in the deleted lists
|
||||
checkBlocks := func() {
|
||||
for _, blk := range have {
|
||||
has, err := db.Has(blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("missing block")
|
||||
}
|
||||
|
||||
blk2, err := db.Get(blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(blk.RawData(), blk2.RawData()) {
|
||||
t.Fatal("data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range deleted {
|
||||
has, err := db.Has(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("resurrected block")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checkBlocks()
|
||||
|
||||
// check the basePath -- it should contain a directory with name db.{timestamp}, soft-linked
|
||||
// to db and nothing else
|
||||
checkPath := func() {
|
||||
entries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("too many entries; expected %d but got %d", 2, len(entries))
|
||||
}
|
||||
|
||||
var haveDB, haveDBLink bool
|
||||
for _, e := range entries {
|
||||
if e.Name() == "db" {
|
||||
if (e.Type() & os.ModeSymlink) == 0 {
|
||||
t.Fatal("found db, but it's not a symlink")
|
||||
}
|
||||
haveDBLink = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(e.Name(), "db.") {
|
||||
if !e.Type().IsDir() {
|
||||
t.Fatal("found db prefix, but it's not a directory")
|
||||
}
|
||||
haveDB = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if !haveDB {
|
||||
t.Fatal("db directory is missing")
|
||||
}
|
||||
if !haveDBLink {
|
||||
t.Fatal("db link is missing")
|
||||
}
|
||||
}
|
||||
|
||||
checkPath()
|
||||
|
||||
// now do another FullGC to test the double move and following of symlinks
|
||||
if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
checkBlocks()
|
||||
checkPath()
|
||||
|
||||
// reopen the db to make sure our relative link works:
|
||||
err = db.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err = Open(optsF(dbPath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// db.Close() is already deferred
|
||||
|
||||
checkBlocks()
|
||||
}
|
||||
|
||||
func TestMoveNoPrefix(t *testing.T) {
|
||||
testMove(t, DefaultOptions)
|
||||
}
|
||||
|
||||
func TestMoveWithPrefix(t *testing.T) {
|
||||
testMove(t, func(path string) Options {
|
||||
opts := DefaultOptions(path)
|
||||
opts.Prefix = "/prefixed/"
|
||||
return opts
|
||||
})
|
||||
}
|
||||
|
||||
@@ -37,27 +37,7 @@ type BlockstoreIterator interface {
|
||||
|
||||
// BlockstoreGC is a trait for blockstores that support online garbage collection
|
||||
type BlockstoreGC interface {
|
||||
CollectGarbage(options ...BlockstoreGCOption) error
|
||||
}
|
||||
|
||||
// BlockstoreGCOption is a functional interface for controlling blockstore GC options
|
||||
type BlockstoreGCOption = func(*BlockstoreGCOptions) error
|
||||
|
||||
// BlockstoreGCOptions is a struct with GC options
|
||||
type BlockstoreGCOptions struct {
|
||||
FullGC bool
|
||||
}
|
||||
|
||||
func WithFullGC(fullgc bool) BlockstoreGCOption {
|
||||
return func(opts *BlockstoreGCOptions) error {
|
||||
opts.FullGC = fullgc
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// BlockstoreSize is a trait for on-disk blockstores that can report their size
|
||||
type BlockstoreSize interface {
|
||||
Size() (int64, error)
|
||||
CollectGarbage() error
|
||||
}
|
||||
|
||||
// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
|
||||
|
||||
@@ -27,47 +27,9 @@ If you intend to use the discard coldstore, your also need to add the following:
|
||||
ColdStoreType = "discard"
|
||||
```
|
||||
In general you _should not_ have to use the discard store, unless you
|
||||
are running a network assistive node (like a bootstrapper or booster)
|
||||
or have very constrained hardware with not enough disk space to
|
||||
maintain a coldstore, even with garbage collection. It is also appropriate
|
||||
for small nodes that are simply watching the chain.
|
||||
|
||||
*Warning:* Using the discard store for a general purpose node is discouraged, unless
|
||||
you really know what you are doing. Use it at your own risk.
|
||||
|
||||
## Configuration Options
|
||||
|
||||
These are options in the `[Chainstore.Splitstore]` section of the configuration:
|
||||
|
||||
- `HotStoreType` -- specifies the type of hotstore to use.
|
||||
The only currently supported option is `"badger"`.
|
||||
- `ColdStoreType` -- specifies the type of coldstore to use.
|
||||
The default value is `"universal"`, which will use the initial monolith blockstore
|
||||
as the coldstore.
|
||||
The other possible value is `"discard"`, as outlined above, which is specialized for
|
||||
running without a coldstore. Note that the discard store wraps the initial monolith
|
||||
blockstore and discards writes; this is necessary to support syncing from a snapshot.
|
||||
- `MarkSetType` -- specifies the type of markset to use during compaction.
|
||||
The markset is the data structure used by compaction/gc to track live objects.
|
||||
The default value is `"map"`, which will use an in-memory map; if you are limited
|
||||
in memory (or indeed see compaction run out of memory), you can also specify
|
||||
`"badger"` which will use an disk backed markset, using badger. This will use
|
||||
much less memory, but will also make compaction slower.
|
||||
- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
|
||||
finalities maintained by default, to maintain messages and message receipts in the
|
||||
hotstore. This is useful for assistive nodes that want to support syncing for other
|
||||
nodes beyond 4 finalities, while running with the discard coldstore option.
|
||||
It is also useful for miners who accept deals and need to lookback messages beyond
|
||||
the 4 finalities, which would otherwise hit the coldstore.
|
||||
- `HotStoreFullGCFrequency` -- specifies how frequenty to garbage collect the hotstore
|
||||
using full (moving) GC.
|
||||
The default value is 20, which uses full GC every 20 compactions (about once a week);
|
||||
set to 0 to disable full GC altogether.
|
||||
Rationale: badger supports online GC, and this is used by default. However it has proven to
|
||||
be ineffective in practice with the hotstore size slowly creeping up. In order to address this,
|
||||
we have added moving GC support in our badger wrapper, which can effectively reclaim all space.
|
||||
The downside is that it takes a bit longer to perform a moving GC and you also need enough
|
||||
space to house the new hotstore while the old one is still live.
|
||||
are running a network booster or have very constrained hardware with
|
||||
not enough disk space to maintain a coldstore, even with garbage
|
||||
collection.
|
||||
|
||||
|
||||
## Operation
|
||||
@@ -105,21 +67,6 @@ Compaction works transactionally with the following algorithm:
|
||||
- We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
|
||||
- We then end the transaction and compact/gc the hotstore.
|
||||
|
||||
## Garbage Collection
|
||||
## Coldstore Garbage Collection
|
||||
|
||||
TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
|
||||
|
||||
## Utilities
|
||||
|
||||
`lotus-shed` has a `splitstore` command which provides some utilities:
|
||||
|
||||
- `rollback` -- rolls back a splitstore installation.
|
||||
This command copies the hotstore on top of the coldstore, and then deletes the splitstore
|
||||
directory and associated metadata keys.
|
||||
It can also optionally compact/gc the coldstore after the copy (with the `--gc-coldstore` flag)
|
||||
and automatically rewrite the lotus config to disable splitstore (with the `--rewrite-config` flag).
|
||||
Note: the node *must be stopped* before running this command.
|
||||
- `clear` -- clears a splitstore installation for restart from snapshot.
|
||||
- `check` -- asynchronously runs a basic healthcheck on the splitstore.
|
||||
The results are appended to `<lotus-repo>/datastore/splitstore/check.txt`.
|
||||
- `info` -- prints some basic information about the splitstore.
|
||||
|
||||
@@ -21,30 +21,17 @@ type MarkSet interface {
|
||||
SetConcurrent()
|
||||
}
|
||||
|
||||
type MarkSetVisitor interface {
|
||||
MarkSet
|
||||
ObjectVisitor
|
||||
}
|
||||
|
||||
type MarkSetEnv interface {
|
||||
// Create creates a new markset within the environment.
|
||||
// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
|
||||
// sizeHint is a hint about the expected size of the markset
|
||||
Create(name string, sizeHint int64) (MarkSet, error)
|
||||
// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
|
||||
// It may not be supported by some markset types (e.g. bloom).
|
||||
CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
|
||||
// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
|
||||
SupportsVisitor() bool
|
||||
Close() error
|
||||
}
|
||||
|
||||
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
|
||||
switch mtype {
|
||||
case "bloom":
|
||||
return NewBloomMarkSetEnv()
|
||||
case "map":
|
||||
return NewMapMarkSetEnv()
|
||||
case "badger":
|
||||
return NewBadgerMarkSetEnv(path)
|
||||
default:
|
||||
return nil, xerrors.Errorf("unknown mark set type %s", mtype)
|
||||
}
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/dgraph-io/badger/v2"
|
||||
"github.com/dgraph-io/badger/v2/options"
|
||||
"go.uber.org/zap"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type BadgerMarkSetEnv struct {
|
||||
path string
|
||||
}
|
||||
|
||||
var _ MarkSetEnv = (*BadgerMarkSetEnv)(nil)
|
||||
|
||||
type BadgerMarkSet struct {
|
||||
mx sync.RWMutex
|
||||
cond sync.Cond
|
||||
pend map[string]struct{}
|
||||
writing map[int]map[string]struct{}
|
||||
writers int
|
||||
seqno int
|
||||
version int
|
||||
|
||||
db *badger.DB
|
||||
path string
|
||||
}
|
||||
|
||||
var _ MarkSet = (*BadgerMarkSet)(nil)
|
||||
var _ MarkSetVisitor = (*BadgerMarkSet)(nil)
|
||||
|
||||
var badgerMarkSetBatchSize = 16384
|
||||
|
||||
func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
|
||||
msPath := filepath.Join(path, "markset.badger")
|
||||
err := os.MkdirAll(msPath, 0755) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating markset directory: %w", err)
|
||||
}
|
||||
|
||||
return &BadgerMarkSetEnv{path: msPath}, nil
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
|
||||
name += ".tmp"
|
||||
path := filepath.Join(e.path, name)
|
||||
|
||||
db, err := openTransientBadgerDB(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating badger db: %w", err)
|
||||
}
|
||||
|
||||
ms := &BadgerMarkSet{
|
||||
pend: make(map[string]struct{}),
|
||||
writing: make(map[int]map[string]struct{}),
|
||||
db: db,
|
||||
path: path,
|
||||
}
|
||||
ms.cond.L = &ms.mx
|
||||
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
return e.create(name, sizeHint)
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
|
||||
return e.create(name, sizeHint)
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
|
||||
|
||||
func (e *BadgerMarkSetEnv) Close() error {
|
||||
return os.RemoveAll(e.path)
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Mark(c cid.Cid) error {
|
||||
s.mx.Lock()
|
||||
if s.pend == nil {
|
||||
s.mx.Unlock()
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
write, seqno := s.put(string(c.Hash()))
|
||||
s.mx.Unlock()
|
||||
|
||||
if write {
|
||||
return s.write(seqno)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
|
||||
key := c.Hash()
|
||||
pendKey := string(key)
|
||||
|
||||
has, err := s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
}
|
||||
|
||||
return s.tryDB(key)
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Visit(c cid.Cid) (bool, error) {
|
||||
key := c.Hash()
|
||||
pendKey := string(key)
|
||||
|
||||
s.mx.RLock()
|
||||
|
||||
has, err := s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
s.mx.RUnlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
has, err = s.tryDB(key)
|
||||
if has || err != nil {
|
||||
s.mx.RUnlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
// we need to upgrade the lock to exclusive in order to write; take the version count to see
|
||||
// if there was another write while we were upgrading
|
||||
version := s.version
|
||||
s.mx.RUnlock()
|
||||
|
||||
s.mx.Lock()
|
||||
// we have to do the check dance again
|
||||
has, err = s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
s.mx.Unlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
if version != s.version {
|
||||
// something was written to the db, we need to check it
|
||||
has, err = s.tryDB(key)
|
||||
if has || err != nil {
|
||||
s.mx.Unlock()
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
write, seqno := s.put(pendKey)
|
||||
s.mx.Unlock()
|
||||
|
||||
if write {
|
||||
err = s.write(seqno)
|
||||
}
|
||||
|
||||
return true, err
|
||||
}
|
||||
|
||||
// reader holds the (r)lock
|
||||
func (s *BadgerMarkSet) tryPending(key string) (has bool, err error) {
|
||||
if s.pend == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
if _, ok := s.pend[key]; ok {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
for _, wr := range s.writing {
|
||||
if _, ok := wr[key]; ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
|
||||
err = s.db.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(key)
|
||||
return err
|
||||
})
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
return true, nil
|
||||
|
||||
case badger.ErrKeyNotFound:
|
||||
return false, nil
|
||||
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// writer holds the exclusive lock
|
||||
func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
|
||||
s.pend[key] = struct{}{}
|
||||
if len(s.pend) < badgerMarkSetBatchSize {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
seqno = s.seqno
|
||||
s.seqno++
|
||||
s.writing[seqno] = s.pend
|
||||
s.pend = make(map[string]struct{})
|
||||
|
||||
return true, seqno
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) write(seqno int) (err error) {
|
||||
s.mx.Lock()
|
||||
if s.pend == nil {
|
||||
s.mx.Unlock()
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
pend := s.writing[seqno]
|
||||
s.writers++
|
||||
s.mx.Unlock()
|
||||
|
||||
defer func() {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if err == nil {
|
||||
delete(s.writing, seqno)
|
||||
s.version++
|
||||
}
|
||||
|
||||
s.writers--
|
||||
if s.writers == 0 {
|
||||
s.cond.Broadcast()
|
||||
}
|
||||
}()
|
||||
|
||||
empty := []byte{} // not nil
|
||||
|
||||
batch := s.db.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for k := range pend {
|
||||
if err = batch.Set([]byte(k), empty); err != nil {
|
||||
return xerrors.Errorf("error setting batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = batch.Flush()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error flushing batch to badger markset: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Close() error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if s.pend == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
for s.writers > 0 {
|
||||
s.cond.Wait()
|
||||
}
|
||||
|
||||
s.pend = nil
|
||||
db := s.db
|
||||
s.db = nil
|
||||
|
||||
return closeTransientBadgerDB(db, s.path)
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) SetConcurrent() {}
|
||||
|
||||
func openTransientBadgerDB(path string) (*badger.DB, error) {
|
||||
// clean up first
|
||||
err := os.RemoveAll(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
|
||||
}
|
||||
|
||||
err = os.MkdirAll(path, 0755) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating markset directory: %w", err)
|
||||
}
|
||||
|
||||
opts := badger.DefaultOptions(path)
|
||||
opts.SyncWrites = false
|
||||
opts.CompactL0OnClose = false
|
||||
opts.Compression = options.None
|
||||
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
|
||||
// between the system blockstore and the markset.
|
||||
// It was observed that using the default memory mapped option resulted in
|
||||
// significant interference and unacceptably high block validation times once the markset
|
||||
// exceeded 1GB in size.
|
||||
opts.TableLoadingMode = options.FileIO
|
||||
opts.ValueLogLoadingMode = options.FileIO
|
||||
opts.Logger = &badgerLogger{
|
||||
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
|
||||
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
|
||||
}
|
||||
|
||||
return badger.Open(opts)
|
||||
}
|
||||
|
||||
func closeTransientBadgerDB(db *badger.DB, path string) error {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error closing badger markset: %w", err)
|
||||
}
|
||||
|
||||
err = os.RemoveAll(path)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error deleting badger markset: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// badger logging through go-log
|
||||
type badgerLogger struct {
|
||||
*zap.SugaredLogger
|
||||
skip2 *zap.SugaredLogger
|
||||
}
|
||||
|
||||
func (b *badgerLogger) Warningf(format string, args ...interface{}) {}
|
||||
func (b *badgerLogger) Infof(format string, args ...interface{}) {}
|
||||
func (b *badgerLogger) Debugf(format string, args ...interface{}) {}
|
||||
@@ -0,0 +1,107 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
bbloom "github.com/ipfs/bbloom"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
const (
|
||||
BloomFilterMinSize = 10_000_000
|
||||
BloomFilterProbability = 0.01
|
||||
)
|
||||
|
||||
type BloomMarkSetEnv struct{}
|
||||
|
||||
var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
|
||||
|
||||
type BloomMarkSet struct {
|
||||
salt []byte
|
||||
mx sync.RWMutex
|
||||
bf *bbloom.Bloom
|
||||
ts bool
|
||||
}
|
||||
|
||||
var _ MarkSet = (*BloomMarkSet)(nil)
|
||||
|
||||
func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
|
||||
return &BloomMarkSetEnv{}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
size := int64(BloomFilterMinSize)
|
||||
for size < sizeHint {
|
||||
size += BloomFilterMinSize
|
||||
}
|
||||
|
||||
salt := make([]byte, 4)
|
||||
_, err := rand.Read(salt)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error reading salt: %w", err)
|
||||
}
|
||||
|
||||
bf, err := bbloom.New(float64(size), BloomFilterProbability)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating bloom filter: %w", err)
|
||||
}
|
||||
|
||||
return &BloomMarkSet{salt: salt, bf: bf}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
|
||||
hash := cid.Hash()
|
||||
key := make([]byte, len(s.salt)+len(hash))
|
||||
n := copy(key, s.salt)
|
||||
copy(key[n:], hash)
|
||||
rehash := sha256.Sum256(key)
|
||||
return rehash[:]
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Mark(cid cid.Cid) error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
|
||||
if s.bf == nil {
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
s.bf.Add(s.saltedKey(cid))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
if s.ts {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
}
|
||||
|
||||
if s.bf == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
return s.bf.Has(s.saltedKey(cid)), nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Close() error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
s.bf = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) SetConcurrent() {
|
||||
s.ts = true
|
||||
}
|
||||
@@ -18,28 +18,17 @@ type MapMarkSet struct {
|
||||
}
|
||||
|
||||
var _ MarkSet = (*MapMarkSet)(nil)
|
||||
var _ MarkSetVisitor = (*MapMarkSet)(nil)
|
||||
|
||||
func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
|
||||
return &MapMarkSetEnv{}, nil
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
|
||||
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
return &MapMarkSet{
|
||||
set: make(map[string]struct{}, sizeHint),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
return e.create(name, sizeHint)
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
|
||||
return e.create(name, sizeHint)
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
|
||||
|
||||
func (e *MapMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -72,25 +61,6 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
|
||||
if s.set == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
key := string(c.Hash())
|
||||
if _, ok := s.set[key]; ok {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
s.set[key] = struct{}{}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Close() error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
|
||||
@@ -2,7 +2,6 @@ package splitstore
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
@@ -11,31 +10,20 @@ import (
|
||||
|
||||
func TestMapMarkSet(t *testing.T) {
|
||||
testMarkSet(t, "map")
|
||||
testMarkSetVisitor(t, "map")
|
||||
}
|
||||
|
||||
func TestBadgerMarkSet(t *testing.T) {
|
||||
bs := badgerMarkSetBatchSize
|
||||
badgerMarkSetBatchSize = 1
|
||||
t.Cleanup(func() {
|
||||
badgerMarkSetBatchSize = bs
|
||||
})
|
||||
testMarkSet(t, "badger")
|
||||
testMarkSetVisitor(t, "badger")
|
||||
func TestBloomMarkSet(t *testing.T) {
|
||||
testMarkSet(t, "bloom")
|
||||
}
|
||||
|
||||
func testMarkSet(t *testing.T, lsType string) {
|
||||
t.Helper()
|
||||
|
||||
path, err := ioutil.TempDir("", "markset.*")
|
||||
path, err := ioutil.TempDir("", "sweep-test.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = os.RemoveAll(path)
|
||||
})
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -148,74 +136,3 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func testMarkSetVisitor(t *testing.T, lsType string) {
|
||||
t.Helper()
|
||||
|
||||
path, err := ioutil.TempDir("", "markset.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = os.RemoveAll(path)
|
||||
})
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
visitor, err := env.CreateVisitor("test", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer visitor.Close() //nolint:errcheck
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
t.Fatal("object should be visited")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if visit {
|
||||
t.Fatal("unexpected visit")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
mustVisit(visitor, k1)
|
||||
mustVisit(visitor, k2)
|
||||
mustVisit(visitor, k3)
|
||||
mustVisit(visitor, k4)
|
||||
|
||||
mustNotVisit(visitor, k1)
|
||||
mustNotVisit(visitor, k2)
|
||||
mustNotVisit(visitor, k3)
|
||||
mustNotVisit(visitor, k4)
|
||||
}
|
||||
|
||||
@@ -62,11 +62,8 @@ func init() {
|
||||
type Config struct {
|
||||
// MarkSetType is the type of mark set to use.
|
||||
//
|
||||
// The default value is "map", which uses an in-memory map-backed markset.
|
||||
// If you are constrained in memory (i.e. compaction runs out of memory), you
|
||||
// can use "badger", which will use a disk-backed markset using badger.
|
||||
// Note that compaction will take quite a bit longer when using the "badger" option,
|
||||
// but that shouldn't really matter (as long as it is under 7.5hrs).
|
||||
// Only current sane value is "map", but we may add an option for a disk-backed
|
||||
// markset for memory-constrained situations.
|
||||
MarkSetType string
|
||||
|
||||
// DiscardColdBlocks indicates whether to skip moving cold blocks to the coldstore.
|
||||
@@ -74,20 +71,6 @@ type Config struct {
|
||||
// which skips moving (as it is a noop, but still takes time to read all the cold objects)
|
||||
// and directly purges cold blocks.
|
||||
DiscardColdBlocks bool
|
||||
|
||||
// HotstoreMessageRetention indicates the hotstore retention policy for messages.
|
||||
// It has the following semantics:
|
||||
// - a value of 0 will only retain messages within the compaction boundary (4 finalities)
|
||||
// - a positive integer indicates the number of finalities, outside the compaction boundary,
|
||||
// for which messages will be retained in the hotstore.
|
||||
HotStoreMessageRetention uint64
|
||||
|
||||
// HotstoreFullGCFrequency indicates how frequently (in terms of compactions) to garbage collect
|
||||
// the hotstore using full (moving) GC if supported by the hotstore.
|
||||
// A value of 0 disables full GC entirely.
|
||||
// A positive value is the number of compactions before a full GC is performed;
|
||||
// a value of 1 will perform full GC in every compaction.
|
||||
HotStoreFullGCFrequency uint64
|
||||
}
|
||||
|
||||
// ChainAccessor allows the Splitstore to access the chain. It will most likely
|
||||
@@ -109,8 +92,7 @@ type SplitStore struct {
|
||||
compacting int32 // compaction/prune/warmup in progress
|
||||
closing int32 // the splitstore is closing
|
||||
|
||||
cfg *Config
|
||||
path string
|
||||
cfg *Config
|
||||
|
||||
mx sync.Mutex
|
||||
warmupEpoch abi.ChainEpoch // protected by mx
|
||||
@@ -142,12 +124,10 @@ type SplitStore struct {
|
||||
txnViews int
|
||||
txnViewsWaiting bool
|
||||
txnActive bool
|
||||
txnProtect MarkSet
|
||||
txnRefsMx sync.Mutex
|
||||
txnRefs map[cid.Cid]struct{}
|
||||
txnMissing map[cid.Cid]struct{}
|
||||
|
||||
// registered protectors
|
||||
protectors []func(func(cid.Cid) error) error
|
||||
}
|
||||
|
||||
var _ bstore.Blockstore = (*SplitStore)(nil)
|
||||
@@ -173,14 +153,9 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !markSetEnv.SupportsVisitor() {
|
||||
return nil, xerrors.Errorf("markset type does not support atomic visitors")
|
||||
}
|
||||
|
||||
// and now we can make a SplitStore
|
||||
ss := &SplitStore{
|
||||
cfg: cfg,
|
||||
path: path,
|
||||
ds: ds,
|
||||
cold: cold,
|
||||
hot: hots,
|
||||
@@ -545,13 +520,6 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) AddProtector(protector func(func(cid.Cid) error) error) {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
s.protectors = append(s.protectors, protector)
|
||||
}
|
||||
|
||||
func (s *SplitStore) Close() error {
|
||||
if !atomic.CompareAndSwapInt32(&s.closing, 0, 1) {
|
||||
// already closing
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
// performs an asynchronous health-check on the splitstore; results are appended to
|
||||
// <splitstore-path>/check.txt
|
||||
func (s *SplitStore) Check() error {
|
||||
s.headChangeMx.Lock()
|
||||
defer s.headChangeMx.Unlock()
|
||||
|
||||
// try to take compaction lock and inhibit compaction while the health-check is running
|
||||
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
|
||||
return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
|
||||
}
|
||||
|
||||
if s.compactionIndex == 0 {
|
||||
atomic.StoreInt32(&s.compacting, 0)
|
||||
return xerrors.Errorf("splitstore hasn't compacted yet; health check is not meaningful")
|
||||
}
|
||||
|
||||
// check if we are actually closing first
|
||||
if err := s.checkClosing(); err != nil {
|
||||
atomic.StoreInt32(&s.compacting, 0)
|
||||
return err
|
||||
}
|
||||
|
||||
curTs := s.chain.GetHeaviestTipSet()
|
||||
go func() {
|
||||
defer atomic.StoreInt32(&s.compacting, 0)
|
||||
|
||||
log.Info("checking splitstore health")
|
||||
start := time.Now()
|
||||
|
||||
err := s.doCheck(curTs)
|
||||
if err != nil {
|
||||
log.Errorf("error checking splitstore health: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infow("health check done", "took", time.Since(start))
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
currentEpoch := curTs.Height()
|
||||
boundaryEpoch := currentEpoch - CompactionBoundary
|
||||
|
||||
outputPath := filepath.Join(s.path, "check.txt")
|
||||
output, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error opening check output file %s: %w", outputPath, err)
|
||||
}
|
||||
defer output.Close() //nolint:errcheck
|
||||
|
||||
write := func(format string, args ...interface{}) {
|
||||
_, err := fmt.Fprintf(output, format+"\n", args...)
|
||||
if err != nil {
|
||||
log.Warnf("error writing check output: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
ts, _ := time.Now().MarshalText()
|
||||
write("---------------------------------------------")
|
||||
write("start check at %s", ts)
|
||||
write("current epoch: %d", currentEpoch)
|
||||
write("boundary epoch: %d", boundaryEpoch)
|
||||
write("compaction index: %d", s.compactionIndex)
|
||||
write("--")
|
||||
|
||||
var coldCnt, missingCnt int64
|
||||
|
||||
visitor, err := s.markSetEnv.CreateVisitor("check", 0)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating visitor: %w", err)
|
||||
}
|
||||
defer visitor.Close() //nolint
|
||||
|
||||
err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
has, err := s.hot.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking hotstore: %w", err)
|
||||
}
|
||||
|
||||
if has {
|
||||
return nil
|
||||
}
|
||||
|
||||
has, err = s.cold.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking coldstore: %w", err)
|
||||
}
|
||||
|
||||
if has {
|
||||
coldCnt++
|
||||
write("cold object reference: %s", c)
|
||||
} else {
|
||||
missingCnt++
|
||||
write("missing object reference: %s", c)
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
err = xerrors.Errorf("error walking chain: %w", err)
|
||||
write("ERROR: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
|
||||
write("--")
|
||||
write("cold: %d missing: %d", coldCnt, missingCnt)
|
||||
write("DONE")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// provides some basic information about the splitstore
|
||||
func (s *SplitStore) Info() map[string]interface{} {
|
||||
info := make(map[string]interface{})
|
||||
info["base epoch"] = s.baseEpoch
|
||||
info["warmup epoch"] = s.warmupEpoch
|
||||
info["compactions"] = s.compactionIndex
|
||||
|
||||
sizer, ok := s.hot.(bstore.BlockstoreSize)
|
||||
if ok {
|
||||
size, err := sizer.Size()
|
||||
if err != nil {
|
||||
log.Warnf("error getting hotstore size: %s", err)
|
||||
} else {
|
||||
info["hotstore size"] = size
|
||||
}
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
@@ -184,6 +184,16 @@ func (s *SplitStore) trackTxnRef(c cid.Cid) {
|
||||
return
|
||||
}
|
||||
|
||||
if s.txnProtect != nil {
|
||||
mark, err := s.txnProtect.Has(c)
|
||||
if err != nil {
|
||||
log.Warnf("error checking markset: %s", err)
|
||||
// track it anyways
|
||||
} else if mark {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
s.txnRefsMx.Lock()
|
||||
s.txnRefs[c] = struct{}{}
|
||||
s.txnRefsMx.Unlock()
|
||||
@@ -199,11 +209,27 @@ func (s *SplitStore) trackTxnRefMany(cids []cid.Cid) {
|
||||
s.txnRefsMx.Lock()
|
||||
defer s.txnRefsMx.Unlock()
|
||||
|
||||
quiet := false
|
||||
for _, c := range cids {
|
||||
if isUnitaryObject(c) {
|
||||
continue
|
||||
}
|
||||
|
||||
if s.txnProtect != nil {
|
||||
mark, err := s.txnProtect.Has(c)
|
||||
if err != nil {
|
||||
if !quiet {
|
||||
quiet = true
|
||||
log.Warnf("error checking markset: %s", err)
|
||||
}
|
||||
// track it anyways
|
||||
}
|
||||
|
||||
if mark {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
s.txnRefs[c] = struct{}{}
|
||||
}
|
||||
|
||||
@@ -211,7 +237,7 @@ func (s *SplitStore) trackTxnRefMany(cids []cid.Cid) {
|
||||
}
|
||||
|
||||
// protect all pending transactional references
|
||||
func (s *SplitStore) protectTxnRefs(markSet MarkSetVisitor) error {
|
||||
func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
|
||||
for {
|
||||
var txnRefs map[cid.Cid]struct{}
|
||||
|
||||
@@ -283,29 +309,30 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSetVisitor) error {
|
||||
|
||||
// transactionally protect a reference by walking the object and marking.
|
||||
// concurrent markings are short circuited by checking the markset.
|
||||
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSetVisitor) error {
|
||||
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Note: cold objects are deleted heaviest first, so the consituents of an object
|
||||
// cannot be deleted before the object itself.
|
||||
return s.walkObjectIncomplete(root, tmpVisitor(),
|
||||
return s.walkObjectIncomplete(root, cid.NewSet(),
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
visit, err := markSet.Visit(c)
|
||||
mark, err := markSet.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error visiting object: %w", err)
|
||||
return xerrors.Errorf("error checking markset: %w", err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
// it's marked, nothing to do
|
||||
if mark {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
return nil
|
||||
return markSet.Mark(c)
|
||||
},
|
||||
func(c cid.Cid) error {
|
||||
if s.txnMissing != nil {
|
||||
@@ -318,30 +345,6 @@ func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSetVisitor) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *SplitStore) applyProtectors() error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
count := 0
|
||||
for _, protect := range s.protectors {
|
||||
err := protect(func(c cid.Cid) error {
|
||||
s.trackTxnRef(c)
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error applynig protector: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
log.Infof("protected %d references through %d protectors", count, len(s.protectors))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Compaction ---
|
||||
// Compaction works transactionally with the following algorithm:
|
||||
// - We prepare a transaction, whereby all i/o referenced objects through the API are tracked.
|
||||
@@ -373,15 +376,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
|
||||
currentEpoch := curTs.Height()
|
||||
boundaryEpoch := currentEpoch - CompactionBoundary
|
||||
|
||||
var inclMsgsEpoch abi.ChainEpoch
|
||||
inclMsgsRange := abi.ChainEpoch(s.cfg.HotStoreMessageRetention) * build.Finality
|
||||
if inclMsgsRange < boundaryEpoch {
|
||||
inclMsgsEpoch = boundaryEpoch - inclMsgsRange
|
||||
}
|
||||
log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "compactionIndex", s.compactionIndex)
|
||||
|
||||
log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "inclMsgsEpoch", inclMsgsEpoch, "compactionIndex", s.compactionIndex)
|
||||
|
||||
markSet, err := s.markSetEnv.CreateVisitor("live", s.markSetSize)
|
||||
markSet, err := s.markSetEnv.Create("live", s.markSetSize)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating mark set: %w", err)
|
||||
}
|
||||
@@ -395,37 +392,20 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
|
||||
// we are ready for concurrent marking
|
||||
s.beginTxnMarking(markSet)
|
||||
|
||||
// 0. track all protected references at beginning of compaction; anything added later should
|
||||
// be transactionally protected by the write
|
||||
log.Info("protecting references with registered protectors")
|
||||
err = s.applyProtectors()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1. mark reachable objects by walking the chain from the current epoch; we keep state roots
|
||||
// and messages until the boundary epoch.
|
||||
log.Info("marking reachable objects")
|
||||
startMark := time.Now()
|
||||
|
||||
var count int64
|
||||
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
|
||||
err = s.walkChain(curTs, boundaryEpoch, true,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
visit, err := markSet.Visit(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error visiting object: %w", err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
count++
|
||||
return nil
|
||||
return markSet.Mark(c)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -586,8 +566,12 @@ func (s *SplitStore) beginTxnProtect() {
|
||||
s.txnMissing = make(map[cid.Cid]struct{})
|
||||
}
|
||||
|
||||
func (s *SplitStore) beginTxnMarking(markSet MarkSetVisitor) {
|
||||
func (s *SplitStore) beginTxnMarking(markSet MarkSet) {
|
||||
markSet.SetConcurrent()
|
||||
|
||||
s.txnLk.Lock()
|
||||
s.txnProtect = markSet
|
||||
s.txnLk.Unlock()
|
||||
}
|
||||
|
||||
func (s *SplitStore) endTxnProtect() {
|
||||
@@ -598,22 +582,27 @@ func (s *SplitStore) endTxnProtect() {
|
||||
return
|
||||
}
|
||||
|
||||
// release markset memory
|
||||
if s.txnProtect != nil {
|
||||
_ = s.txnProtect.Close()
|
||||
}
|
||||
|
||||
s.txnActive = false
|
||||
s.txnProtect = nil
|
||||
s.txnRefs = nil
|
||||
s.txnMissing = nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
|
||||
visitor ObjectVisitor, f func(cid.Cid) error) error {
|
||||
var walked *cid.Set
|
||||
func (s *SplitStore) walkChain(ts *types.TipSet, boundary abi.ChainEpoch, inclMsgs bool,
|
||||
f func(cid.Cid) error) error {
|
||||
visited := cid.NewSet()
|
||||
walked := cid.NewSet()
|
||||
toWalk := ts.Cids()
|
||||
walkCnt := 0
|
||||
scanCnt := 0
|
||||
|
||||
stopWalk := func(_ cid.Cid) error { return errStopWalk }
|
||||
|
||||
walkBlock := func(c cid.Cid) error {
|
||||
if !walked.Visit(c) {
|
||||
if !visited.Visit(c) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -632,35 +621,22 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
|
||||
return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
|
||||
}
|
||||
|
||||
// message are retained if within the inclMsgs boundary
|
||||
if hdr.Height >= inclMsgs && hdr.Height > 0 {
|
||||
if inclMsgs < inclState {
|
||||
// we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we
|
||||
// synced from snapshot and have a long HotStoreMessageRetentionPolicy.
|
||||
if err := s.walkObjectIncomplete(hdr.Messages, visitor, f, stopWalk); err != nil {
|
||||
// we only scan the block if it is at or above the boundary
|
||||
if hdr.Height >= boundary || hdr.Height == 0 {
|
||||
scanCnt++
|
||||
if inclMsgs && hdr.Height > 0 {
|
||||
if err := s.walkObject(hdr.Messages, walked, f); err != nil {
|
||||
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
|
||||
}
|
||||
|
||||
if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, f, stopWalk); err != nil {
|
||||
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
|
||||
}
|
||||
} else {
|
||||
if err := s.walkObject(hdr.Messages, visitor, f); err != nil {
|
||||
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
|
||||
}
|
||||
|
||||
if err := s.walkObject(hdr.ParentMessageReceipts, visitor, f); err != nil {
|
||||
if err := s.walkObject(hdr.ParentMessageReceipts, walked, f); err != nil {
|
||||
return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// state is only retained if within the inclState boundary, with the exception of genesis
|
||||
if hdr.Height >= inclState || hdr.Height == 0 {
|
||||
if err := s.walkObject(hdr.ParentStateRoot, visitor, f); err != nil {
|
||||
if err := s.walkObject(hdr.ParentStateRoot, walked, f); err != nil {
|
||||
return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
|
||||
}
|
||||
scanCnt++
|
||||
}
|
||||
|
||||
if hdr.Height > 0 {
|
||||
@@ -676,10 +652,6 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
|
||||
return err
|
||||
}
|
||||
|
||||
// the walk is BFS, so we can reset the walked set in every iteration and avoid building up
|
||||
// a set that contains all blocks (1M epochs -> 5M blocks -> 200MB worth of memory and growing
|
||||
// over time)
|
||||
walked = cid.NewSet()
|
||||
walking := toWalk
|
||||
toWalk = nil
|
||||
for _, c := range walking {
|
||||
@@ -694,13 +666,8 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error {
|
||||
visit, err := visitor.Visit(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error visiting object: %w", err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
func (s *SplitStore) walkObject(c cid.Cid, walked *cid.Set, f func(cid.Cid) error) error {
|
||||
if !walked.Visit(c) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -722,7 +689,7 @@ func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid
|
||||
}
|
||||
|
||||
var links []cid.Cid
|
||||
err = s.view(c, func(data []byte) error {
|
||||
err := s.view(c, func(data []byte) error {
|
||||
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
|
||||
links = append(links, c)
|
||||
})
|
||||
@@ -733,7 +700,7 @@ func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid
|
||||
}
|
||||
|
||||
for _, c := range links {
|
||||
err := s.walkObject(c, visitor, f)
|
||||
err := s.walkObject(c, walked, f)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
|
||||
}
|
||||
@@ -743,13 +710,8 @@ func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid
|
||||
}
|
||||
|
||||
// like walkObject, but the object may be potentially incomplete (references missing)
|
||||
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error {
|
||||
visit, err := visitor.Visit(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error visiting object: %w", err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, walked *cid.Set, f, missing func(cid.Cid) error) error {
|
||||
if !walked.Visit(c) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -788,7 +750,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
|
||||
}
|
||||
|
||||
var links []cid.Cid
|
||||
err = s.view(c, func(data []byte) error {
|
||||
err := s.view(c, func(data []byte) error {
|
||||
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
|
||||
links = append(links, c)
|
||||
})
|
||||
@@ -799,7 +761,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
|
||||
}
|
||||
|
||||
for _, c := range links {
|
||||
err := s.walkObjectIncomplete(c, visitor, f, missing)
|
||||
err := s.walkObjectIncomplete(c, walked, f, missing)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
|
||||
}
|
||||
@@ -995,7 +957,7 @@ func (s *SplitStore) purgeBatch(cids []cid.Cid, deleteBatch func([]cid.Cid) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSetVisitor) error {
|
||||
func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSet) error {
|
||||
deadCids := make([]cid.Cid, 0, batchSize)
|
||||
var purgeCnt, liveCnt int
|
||||
defer func() {
|
||||
@@ -1061,7 +1023,7 @@ func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSetVisitor) error {
|
||||
// have this gem[TM].
|
||||
// My best guess is that they are parent message receipts or yet to be computed state roots; magik
|
||||
// thinks the cause may be block validation.
|
||||
func (s *SplitStore) waitForMissingRefs(markSet MarkSetVisitor) {
|
||||
func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
|
||||
s.txnLk.Lock()
|
||||
missing := s.txnMissing
|
||||
s.txnMissing = nil
|
||||
@@ -1090,27 +1052,27 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSetVisitor) {
|
||||
}
|
||||
|
||||
towalk := missing
|
||||
visitor := tmpVisitor()
|
||||
walked := cid.NewSet()
|
||||
missing = make(map[cid.Cid]struct{})
|
||||
|
||||
for c := range towalk {
|
||||
err := s.walkObjectIncomplete(c, visitor,
|
||||
err := s.walkObjectIncomplete(c, walked,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
visit, err := markSet.Visit(c)
|
||||
mark, err := markSet.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error visiting object: %w", err)
|
||||
return xerrors.Errorf("error checking markset for %s: %w", c, err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
if mark {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
count++
|
||||
return nil
|
||||
return markSet.Mark(c)
|
||||
},
|
||||
func(c cid.Cid) error {
|
||||
missing[c] = struct{}{}
|
||||
|
||||
@@ -8,22 +8,17 @@ import (
|
||||
)
|
||||
|
||||
func (s *SplitStore) gcHotstore() {
|
||||
var opts []bstore.BlockstoreGCOption
|
||||
if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
|
||||
opts = append(opts, bstore.WithFullGC(true))
|
||||
}
|
||||
|
||||
if err := s.gcBlockstore(s.hot, opts); err != nil {
|
||||
if err := s.gcBlockstoreOnline(s.hot); err != nil {
|
||||
log.Warnf("error garbage collecting hostore: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
|
||||
func (s *SplitStore) gcBlockstoreOnline(b bstore.Blockstore) error {
|
||||
if gc, ok := b.(bstore.BlockstoreGC); ok {
|
||||
log.Info("garbage collecting blockstore")
|
||||
startGC := time.Now()
|
||||
|
||||
if err := gc.CollectGarbage(opts...); err != nil {
|
||||
if err := gc.CollectGarbage(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -31,5 +26,5 @@ func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreG
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
|
||||
return fmt.Errorf("blockstore doesn't support online gc: %T", b)
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
func init() {
|
||||
CompactionThreshold = 5
|
||||
CompactionBoundary = 2
|
||||
WarmupBoundary = 0
|
||||
logging.SetLogLevel("splitstore", "DEBUG")
|
||||
}
|
||||
|
||||
@@ -64,20 +63,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// create a garbage block that is protected with a rgistered protector
|
||||
protected := blocks.NewBlock([]byte("protected!"))
|
||||
err = hot.Put(protected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// and another one that is not protected
|
||||
unprotected := blocks.NewBlock([]byte("unprotected!"))
|
||||
err = hot.Put(unprotected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// open the splitstore
|
||||
ss, err := Open("", ds, hot, cold, cfg)
|
||||
if err != nil {
|
||||
@@ -85,11 +70,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
}
|
||||
defer ss.Close() //nolint
|
||||
|
||||
// register our protector
|
||||
ss.AddProtector(func(protect func(cid.Cid) error) error {
|
||||
return protect(protected.Cid())
|
||||
})
|
||||
|
||||
err = ss.Start(chain)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -152,8 +132,8 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 12 {
|
||||
t.Errorf("expected %d blocks, but got %d", 12, hotCnt)
|
||||
if hotCnt != 10 {
|
||||
t.Errorf("expected %d blocks, but got %d", 10, hotCnt)
|
||||
}
|
||||
|
||||
// trigger a compaction
|
||||
@@ -166,41 +146,12 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
coldCnt = countBlocks(cold)
|
||||
hotCnt = countBlocks(hot)
|
||||
|
||||
if coldCnt != 6 {
|
||||
t.Errorf("expected %d cold blocks, but got %d", 6, coldCnt)
|
||||
if coldCnt != 5 {
|
||||
t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
|
||||
}
|
||||
|
||||
if hotCnt != 18 {
|
||||
t.Errorf("expected %d hot blocks, but got %d", 18, hotCnt)
|
||||
}
|
||||
|
||||
// ensure our protected block is still there
|
||||
has, err := hot.Has(protected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("protected block is missing from hotstore")
|
||||
}
|
||||
|
||||
// ensure our unprotected block is in the coldstore now
|
||||
has, err = hot.Has(unprotected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("unprotected block is still in hotstore")
|
||||
}
|
||||
|
||||
has, err = cold.Has(unprotected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("unprotected block is missing from coldstore")
|
||||
if hotCnt != 17 {
|
||||
t.Errorf("expected %d hot blocks, but got %d", 17, hotCnt)
|
||||
}
|
||||
|
||||
// Make sure we can revert without panicking.
|
||||
@@ -211,15 +162,6 @@ func TestSplitStoreCompaction(t *testing.T) {
|
||||
testSplitStore(t, &Config{MarkSetType: "map"})
|
||||
}
|
||||
|
||||
func TestSplitStoreCompactionWithBadger(t *testing.T) {
|
||||
bs := badgerMarkSetBatchSize
|
||||
badgerMarkSetBatchSize = 1
|
||||
t.Cleanup(func() {
|
||||
badgerMarkSetBatchSize = bs
|
||||
})
|
||||
testSplitStore(t, &Config{MarkSetType: "badger"})
|
||||
}
|
||||
|
||||
type mockChain struct {
|
||||
t testing.TB
|
||||
|
||||
|
||||
@@ -9,17 +9,10 @@ import (
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
var (
|
||||
// WarmupBoundary is the number of epochs to load state during warmup.
|
||||
WarmupBoundary = build.Finality
|
||||
)
|
||||
|
||||
// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
|
||||
// this is necessary when we sync from a snapshot or when we enable the splitstore
|
||||
// on top of an existing blockstore (which becomes the coldstore).
|
||||
@@ -50,24 +43,12 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error {
|
||||
// and headers all the way up to genesis.
|
||||
// objects are written in batches so as to minimize overhead.
|
||||
func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
|
||||
var boundaryEpoch abi.ChainEpoch
|
||||
epoch := curTs.Height()
|
||||
if WarmupBoundary < epoch {
|
||||
boundaryEpoch = epoch - WarmupBoundary
|
||||
}
|
||||
batchHot := make([]blocks.Block, 0, batchSize)
|
||||
count := int64(0)
|
||||
xcount := int64(0)
|
||||
missing := int64(0)
|
||||
|
||||
visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating visitor: %w", err)
|
||||
}
|
||||
defer visitor.Close() //nolint
|
||||
|
||||
err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
|
||||
visitor,
|
||||
err := s.walkChain(curTs, epoch, false,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
@@ -88,7 +69,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
|
||||
if err != nil {
|
||||
if err == bstore.ErrNotFound {
|
||||
missing++
|
||||
return errStopWalk
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// ObjectVisitor is an interface for deduplicating objects during walks
|
||||
type ObjectVisitor interface {
|
||||
Visit(cid.Cid) (bool, error)
|
||||
}
|
||||
|
||||
type noopVisitor struct{}
|
||||
|
||||
var _ ObjectVisitor = (*noopVisitor)(nil)
|
||||
|
||||
func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type cidSetVisitor struct {
|
||||
set *cid.Set
|
||||
}
|
||||
|
||||
var _ ObjectVisitor = (*cidSetVisitor)(nil)
|
||||
|
||||
func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
|
||||
return v.set.Visit(c), nil
|
||||
}
|
||||
|
||||
func tmpVisitor() ObjectVisitor {
|
||||
return &cidSetVisitor{set: cid.NewSet()}
|
||||
}
|
||||
+1
-5
@@ -47,12 +47,8 @@ func (t *TimedCacheBlockstore) Start(_ context.Context) error {
|
||||
return fmt.Errorf("already started")
|
||||
}
|
||||
t.closeCh = make(chan struct{})
|
||||
|
||||
// Create this timer before starting the goroutine. Otherwise, creating the timer will race
|
||||
// with adding time to the mock clock, and we could add time _first_, then stall waiting for
|
||||
// a timer that'll never fire.
|
||||
ticker := t.clock.Ticker(t.interval)
|
||||
go func() {
|
||||
ticker := t.clock.Ticker(t.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
package build
|
||||
|
||||
var (
|
||||
DefaultFDLimit uint64 = 16 << 10
|
||||
MinerFDLimit uint64 = 100_000
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+8
-9
@@ -1,4 +1,3 @@
|
||||
//go:build debug || 2k
|
||||
// +build debug 2k
|
||||
|
||||
package build
|
||||
@@ -29,18 +28,18 @@ var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
|
||||
var UpgradeLiftoffHeight = abi.ChainEpoch(-6)
|
||||
|
||||
var UpgradeKumquatHeight = abi.ChainEpoch(-7)
|
||||
var UpgradeCalicoHeight = abi.ChainEpoch(-9)
|
||||
var UpgradePersianHeight = abi.ChainEpoch(-10)
|
||||
var UpgradeOrangeHeight = abi.ChainEpoch(-11)
|
||||
var UpgradeClausHeight = abi.ChainEpoch(-12)
|
||||
var UpgradeCalicoHeight = abi.ChainEpoch(-8)
|
||||
var UpgradePersianHeight = abi.ChainEpoch(-9)
|
||||
var UpgradeOrangeHeight = abi.ChainEpoch(-10)
|
||||
var UpgradeClausHeight = abi.ChainEpoch(-11)
|
||||
|
||||
var UpgradeTrustHeight = abi.ChainEpoch(-13)
|
||||
var UpgradeTrustHeight = abi.ChainEpoch(-12)
|
||||
|
||||
var UpgradeNorwegianHeight = abi.ChainEpoch(-14)
|
||||
var UpgradeNorwegianHeight = abi.ChainEpoch(-13)
|
||||
|
||||
var UpgradeTurboHeight = abi.ChainEpoch(-15)
|
||||
var UpgradeTurboHeight = abi.ChainEpoch(-14)
|
||||
|
||||
var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
|
||||
var UpgradeHyperdriveHeight = abi.ChainEpoch(-15)
|
||||
|
||||
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
|
||||
0: DrandMainnet,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build butterflynet
|
||||
// +build butterflynet
|
||||
|
||||
package build
|
||||
@@ -47,8 +46,6 @@ func init() {
|
||||
SetAddressNetwork(address.Testnet)
|
||||
|
||||
Devnet = true
|
||||
|
||||
BuildType = BuildButterflynet
|
||||
}
|
||||
|
||||
const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build calibnet
|
||||
// +build calibnet
|
||||
|
||||
package build
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build debug
|
||||
// +build debug
|
||||
|
||||
package build
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build interopnet
|
||||
// +build interopnet
|
||||
|
||||
package build
|
||||
@@ -32,18 +31,18 @@ var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
|
||||
var UpgradeLiftoffHeight = abi.ChainEpoch(-6)
|
||||
|
||||
var UpgradeKumquatHeight = abi.ChainEpoch(-7)
|
||||
var UpgradeCalicoHeight = abi.ChainEpoch(-9)
|
||||
var UpgradePersianHeight = abi.ChainEpoch(-10)
|
||||
var UpgradeOrangeHeight = abi.ChainEpoch(-11)
|
||||
var UpgradeClausHeight = abi.ChainEpoch(-12)
|
||||
var UpgradeCalicoHeight = abi.ChainEpoch(-8)
|
||||
var UpgradePersianHeight = abi.ChainEpoch(-9)
|
||||
var UpgradeOrangeHeight = abi.ChainEpoch(-10)
|
||||
var UpgradeClausHeight = abi.ChainEpoch(-11)
|
||||
|
||||
var UpgradeTrustHeight = abi.ChainEpoch(-13)
|
||||
var UpgradeTrustHeight = abi.ChainEpoch(-12)
|
||||
|
||||
var UpgradeNorwegianHeight = abi.ChainEpoch(-14)
|
||||
var UpgradeNorwegianHeight = abi.ChainEpoch(-13)
|
||||
|
||||
var UpgradeTurboHeight = abi.ChainEpoch(-15)
|
||||
var UpgradeTurboHeight = abi.ChainEpoch(-14)
|
||||
|
||||
var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
|
||||
var UpgradeHyperdriveHeight = abi.ChainEpoch(-15)
|
||||
|
||||
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
|
||||
0: DrandMainnet,
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//go:build !debug && !2k && !testground && !calibnet && !nerpanet && !butterflynet && !interopnet
|
||||
// +build !debug,!2k,!testground,!calibnet,!nerpanet,!butterflynet,!interopnet
|
||||
// +build !debug
|
||||
// +build !2k
|
||||
// +build !testground
|
||||
// +build !calibnet
|
||||
// +build !nerpanet
|
||||
// +build !butterflynet
|
||||
// +build !interopnet
|
||||
|
||||
package build
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build nerpanet
|
||||
// +build nerpanet
|
||||
|
||||
package build
|
||||
@@ -43,7 +42,7 @@ const UpgradeOrangeHeight = 300
|
||||
const UpgradeTrustHeight = 600
|
||||
const UpgradeNorwegianHeight = 201000
|
||||
const UpgradeTurboHeight = 203000
|
||||
const UpgradeHyperdriveHeight = 379178
|
||||
const UpgradeHyperdriveHeight = 999999999
|
||||
|
||||
func init() {
|
||||
// Minimum block production power is set to 4 TiB
|
||||
@@ -67,8 +66,6 @@ func init() {
|
||||
//miner.WPoStChallengeLookback = abi.ChainEpoch(2)
|
||||
|
||||
Devnet = false
|
||||
|
||||
BuildType = BuildNerpanet
|
||||
}
|
||||
|
||||
const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build !testground
|
||||
// +build !testground
|
||||
|
||||
package build
|
||||
@@ -27,6 +26,7 @@ const UnixfsLinksPerLevel = 1024
|
||||
|
||||
const AllowableClockDriftSecs = uint64(1)
|
||||
const NewestNetworkVersion = network.Version13
|
||||
const ActorUpgradeNetworkVersion = network.Version4
|
||||
|
||||
// Epochs
|
||||
const ForkLengthThreshold = Finality
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build testground
|
||||
// +build testground
|
||||
|
||||
// This file makes hardcoded parameters (const) configurable as vars.
|
||||
@@ -90,14 +89,14 @@ var (
|
||||
UpgradeAssemblyHeight abi.ChainEpoch = 10
|
||||
UpgradeLiftoffHeight abi.ChainEpoch = -5
|
||||
UpgradeKumquatHeight abi.ChainEpoch = -6
|
||||
UpgradeCalicoHeight abi.ChainEpoch = -8
|
||||
UpgradePersianHeight abi.ChainEpoch = -9
|
||||
UpgradeOrangeHeight abi.ChainEpoch = -10
|
||||
UpgradeClausHeight abi.ChainEpoch = -11
|
||||
UpgradeTrustHeight abi.ChainEpoch = -12
|
||||
UpgradeNorwegianHeight abi.ChainEpoch = -13
|
||||
UpgradeTurboHeight abi.ChainEpoch = -14
|
||||
UpgradeHyperdriveHeight abi.ChainEpoch = -15
|
||||
UpgradeCalicoHeight abi.ChainEpoch = -7
|
||||
UpgradePersianHeight abi.ChainEpoch = -8
|
||||
UpgradeOrangeHeight abi.ChainEpoch = -9
|
||||
UpgradeClausHeight abi.ChainEpoch = -10
|
||||
UpgradeTrustHeight abi.ChainEpoch = -11
|
||||
UpgradeNorwegianHeight abi.ChainEpoch = -12
|
||||
UpgradeTurboHeight abi.ChainEpoch = -13
|
||||
UpgradeHyperdriveHeight abi.ChainEpoch = -13
|
||||
|
||||
DrandSchedule = map[abi.ChainEpoch]DrandEnum{
|
||||
0: DrandMainnet,
|
||||
|
||||
+1
-2
@@ -1,5 +1,4 @@
|
||||
//go:build tools
|
||||
// +build tools
|
||||
//+build tools
|
||||
|
||||
package build
|
||||
|
||||
|
||||
+8
-14
@@ -6,14 +6,12 @@ var CurrentCommit string
|
||||
var BuildType int
|
||||
|
||||
const (
|
||||
BuildDefault = 0
|
||||
BuildMainnet = 0x1
|
||||
Build2k = 0x2
|
||||
BuildDebug = 0x3
|
||||
BuildCalibnet = 0x4
|
||||
BuildInteropnet = 0x5
|
||||
BuildNerpanet = 0x6
|
||||
BuildButterflynet = 0x7
|
||||
BuildDefault = 0
|
||||
BuildMainnet = 0x1
|
||||
Build2k = 0x2
|
||||
BuildDebug = 0x3
|
||||
BuildCalibnet = 0x4
|
||||
BuildInteropnet = 0x5
|
||||
)
|
||||
|
||||
func buildType() string {
|
||||
@@ -30,17 +28,13 @@ func buildType() string {
|
||||
return "+calibnet"
|
||||
case BuildInteropnet:
|
||||
return "+interopnet"
|
||||
case BuildNerpanet:
|
||||
return "+nerpanet"
|
||||
case BuildButterflynet:
|
||||
return "+butterflynet"
|
||||
default:
|
||||
return "+huh?"
|
||||
}
|
||||
}
|
||||
|
||||
// BuildVersion is the local build version
|
||||
const BuildVersion = "1.11.3"
|
||||
// BuildVersion is the local build version, set by build system
|
||||
const BuildVersion = "1.11.1-dev"
|
||||
|
||||
func UserVersion() string {
|
||||
if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
|
||||
|
||||
@@ -38,7 +38,7 @@ func init() {
|
||||
|
||||
var Methods = builtin{{.latestVersion}}.MethodsMiner
|
||||
|
||||
// Unchanged between v0, v2, v3, v4, and v5 actors
|
||||
// Unchanged between v0, v2, v3, and v4 actors
|
||||
var WPoStProvingPeriod = miner0.WPoStProvingPeriod
|
||||
var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines
|
||||
var WPoStChallengeWindow = miner0.WPoStChallengeWindow
|
||||
@@ -105,7 +105,6 @@ type State interface {
|
||||
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
|
||||
// count if there aren't enough).
|
||||
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
GetAllocatedSectors() (*bitfield.BitField, error)
|
||||
|
||||
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
|
||||
GetProvingPeriodStart() (abi.ChainEpoch, error)
|
||||
@@ -141,28 +140,11 @@ type Deadline interface {
|
||||
}
|
||||
|
||||
type Partition interface {
|
||||
// AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors
|
||||
AllSectors() (bitfield.BitField, error)
|
||||
|
||||
// Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt).
|
||||
// Faults ∩ Terminated = ∅
|
||||
FaultySectors() (bitfield.BitField, error)
|
||||
|
||||
// Subset of faulty sectors expected to recover on next PoSt
|
||||
// Recoveries ∩ Terminated = ∅
|
||||
RecoveringSectors() (bitfield.BitField, error)
|
||||
|
||||
// Live sectors are those that are not terminated (but may be faulty).
|
||||
LiveSectors() (bitfield.BitField, error)
|
||||
|
||||
// Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power.
|
||||
ActiveSectors() (bitfield.BitField, error)
|
||||
|
||||
// Unproven sectors in this partition. This bitfield will be cleared on
|
||||
// a successful window post (or at the end of the partition's next
|
||||
// deadline). At that time, any still unproven sectors will be added to
|
||||
// the faulty sector bitfield.
|
||||
UnprovenSectors() (bitfield.BitField, error)
|
||||
}
|
||||
|
||||
type SectorOnChainInfo struct {
|
||||
|
||||
@@ -61,7 +61,7 @@ func init() {
|
||||
|
||||
var Methods = builtin5.MethodsMiner
|
||||
|
||||
// Unchanged between v0, v2, v3, v4, and v5 actors
|
||||
// Unchanged between v0, v2, v3, and v4 actors
|
||||
var WPoStProvingPeriod = miner0.WPoStProvingPeriod
|
||||
var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines
|
||||
var WPoStChallengeWindow = miner0.WPoStChallengeWindow
|
||||
@@ -164,7 +164,6 @@ type State interface {
|
||||
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
|
||||
// count if there aren't enough).
|
||||
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
GetAllocatedSectors() (*bitfield.BitField, error)
|
||||
|
||||
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
|
||||
GetProvingPeriodStart() (abi.ChainEpoch, error)
|
||||
@@ -200,28 +199,11 @@ type Deadline interface {
|
||||
}
|
||||
|
||||
type Partition interface {
|
||||
// AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors
|
||||
AllSectors() (bitfield.BitField, error)
|
||||
|
||||
// Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt).
|
||||
// Faults ∩ Terminated = ∅
|
||||
FaultySectors() (bitfield.BitField, error)
|
||||
|
||||
// Subset of faulty sectors expected to recover on next PoSt
|
||||
// Recoveries ∩ Terminated = ∅
|
||||
RecoveringSectors() (bitfield.BitField, error)
|
||||
|
||||
// Live sectors are those that are not terminated (but may be faulty).
|
||||
LiveSectors() (bitfield.BitField, error)
|
||||
|
||||
// Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power.
|
||||
ActiveSectors() (bitfield.BitField, error)
|
||||
|
||||
// Unproven sectors in this partition. This bitfield will be cleared on
|
||||
// a successful window post (or at the end of the partition's next
|
||||
// deadline). At that time, any still unproven sectors will be added to
|
||||
// the faulty sector bitfield.
|
||||
UnprovenSectors() (bitfield.BitField, error)
|
||||
}
|
||||
|
||||
type SectorOnChainInfo struct {
|
||||
|
||||
@@ -318,15 +318,6 @@ func (s *state{{.v}}) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, e
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state{{.v}}) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state{{.v}}) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -479,15 +470,12 @@ func (s *state{{.v}}) EraseAllUnproven() error {
|
||||
|
||||
return dls.UpdateDeadline(s.store, dindx, dl)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.State.SaveDeadlines(s.store, dls)
|
||||
{{else}}
|
||||
// field doesn't exist until v2
|
||||
return nil
|
||||
{{end}}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline{{.v}}) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -549,10 +537,6 @@ func (p *partition{{.v}}) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition{{.v}}) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return {{if (ge .v 2)}}p.Partition.Unproven{{else}}bitfield.New(){{end}}, nil
|
||||
}
|
||||
|
||||
func fromV{{.v}}SectorOnChainInfo(v{{.v}} miner{{.v}}.SectorOnChainInfo) SectorOnChainInfo {
|
||||
{{if (ge .v 2)}}
|
||||
return SectorOnChainInfo{
|
||||
|
||||
@@ -311,15 +311,6 @@ func (s *state0) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state0) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state0) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -444,8 +435,8 @@ func (s *state0) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreC
|
||||
func (s *state0) EraseAllUnproven() error {
|
||||
|
||||
// field doesn't exist until v2
|
||||
return nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline0) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -500,10 +491,6 @@ func (p *partition0) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition0) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return bitfield.New(), nil
|
||||
}
|
||||
|
||||
func fromV0SectorOnChainInfo(v0 miner0.SectorOnChainInfo) SectorOnChainInfo {
|
||||
|
||||
return (SectorOnChainInfo)(v0)
|
||||
|
||||
@@ -309,15 +309,6 @@ func (s *state2) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state2) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state2) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -470,12 +461,10 @@ func (s *state2) EraseAllUnproven() error {
|
||||
|
||||
return dls.UpdateDeadline(s.store, dindx, dl)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.State.SaveDeadlines(s.store, dls)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline2) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -530,10 +519,6 @@ func (p *partition2) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition2) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Unproven, nil
|
||||
}
|
||||
|
||||
func fromV2SectorOnChainInfo(v2 miner2.SectorOnChainInfo) SectorOnChainInfo {
|
||||
|
||||
return SectorOnChainInfo{
|
||||
|
||||
@@ -311,15 +311,6 @@ func (s *state3) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state3) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state3) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -467,12 +458,10 @@ func (s *state3) EraseAllUnproven() error {
|
||||
|
||||
return dls.UpdateDeadline(s.store, dindx, dl)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.State.SaveDeadlines(s.store, dls)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline3) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -531,10 +520,6 @@ func (p *partition3) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition3) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Unproven, nil
|
||||
}
|
||||
|
||||
func fromV3SectorOnChainInfo(v3 miner3.SectorOnChainInfo) SectorOnChainInfo {
|
||||
|
||||
return SectorOnChainInfo{
|
||||
|
||||
@@ -311,15 +311,6 @@ func (s *state4) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state4) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state4) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -467,12 +458,10 @@ func (s *state4) EraseAllUnproven() error {
|
||||
|
||||
return dls.UpdateDeadline(s.store, dindx, dl)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.State.SaveDeadlines(s.store, dls)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline4) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -531,10 +520,6 @@ func (p *partition4) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition4) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Unproven, nil
|
||||
}
|
||||
|
||||
func fromV4SectorOnChainInfo(v4 miner4.SectorOnChainInfo) SectorOnChainInfo {
|
||||
|
||||
return SectorOnChainInfo{
|
||||
|
||||
@@ -311,15 +311,6 @@ func (s *state5) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
|
||||
return sectors, nil
|
||||
}
|
||||
|
||||
func (s *state5) GetAllocatedSectors() (*bitfield.BitField, error) {
|
||||
var allocatedSectors bitfield.BitField
|
||||
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &allocatedSectors, nil
|
||||
}
|
||||
|
||||
func (s *state5) LoadDeadline(idx uint64) (Deadline, error) {
|
||||
dls, err := s.State.LoadDeadlines(s.store)
|
||||
if err != nil {
|
||||
@@ -467,12 +458,10 @@ func (s *state5) EraseAllUnproven() error {
|
||||
|
||||
return dls.UpdateDeadline(s.store, dindx, dl)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.State.SaveDeadlines(s.store, dls)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *deadline5) LoadPartition(idx uint64) (Partition, error) {
|
||||
@@ -531,10 +520,6 @@ func (p *partition5) RecoveringSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Recoveries, nil
|
||||
}
|
||||
|
||||
func (p *partition5) UnprovenSectors() (bitfield.BitField, error) {
|
||||
return p.Partition.Unproven, nil
|
||||
}
|
||||
|
||||
func fromV5SectorOnChainInfo(v5 miner5.SectorOnChainInfo) SectorOnChainInfo {
|
||||
|
||||
return SectorOnChainInfo{
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
@@ -169,31 +168,31 @@ func SetMinVerifiedDealSize(size abi.StoragePower) {
|
||||
|
||||
}
|
||||
|
||||
func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) (abi.ChainEpoch, error) {
|
||||
func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) abi.ChainEpoch {
|
||||
switch ver {
|
||||
|
||||
case actors.Version0:
|
||||
|
||||
return miner0.MaxSealDuration[t], nil
|
||||
return miner0.MaxSealDuration[t]
|
||||
|
||||
case actors.Version2:
|
||||
|
||||
return miner2.MaxProveCommitDuration[t], nil
|
||||
return miner2.MaxProveCommitDuration[t]
|
||||
|
||||
case actors.Version3:
|
||||
|
||||
return miner3.MaxProveCommitDuration[t], nil
|
||||
return miner3.MaxProveCommitDuration[t]
|
||||
|
||||
case actors.Version4:
|
||||
|
||||
return miner4.MaxProveCommitDuration[t], nil
|
||||
return miner4.MaxProveCommitDuration[t]
|
||||
|
||||
case actors.Version5:
|
||||
|
||||
return miner5.MaxProveCommitDuration[t], nil
|
||||
return miner5.MaxProveCommitDuration[t]
|
||||
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported actors version")
|
||||
panic("unsupported actors version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,40 +227,31 @@ func DealProviderCollateralBounds(
|
||||
size abi.PaddedPieceSize, verified bool,
|
||||
rawBytePower, qaPower, baselinePower abi.StoragePower,
|
||||
circulatingFil abi.TokenAmount, nwVer network.Version,
|
||||
) (min, max abi.TokenAmount, err error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return big.Zero(), big.Zero(), err
|
||||
}
|
||||
switch v {
|
||||
) (min, max abi.TokenAmount) {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
|
||||
case actors.Version0:
|
||||
|
||||
min, max := market0.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil, nwVer)
|
||||
return min, max, nil
|
||||
return market0.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil, nwVer)
|
||||
|
||||
case actors.Version2:
|
||||
|
||||
min, max := market2.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
return min, max, nil
|
||||
return market2.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
|
||||
case actors.Version3:
|
||||
|
||||
min, max := market3.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
return min, max, nil
|
||||
return market3.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
|
||||
case actors.Version4:
|
||||
|
||||
min, max := market4.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
return min, max, nil
|
||||
return market4.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
|
||||
case actors.Version5:
|
||||
|
||||
min, max := market5.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
return min, max, nil
|
||||
return market5.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
|
||||
default:
|
||||
return big.Zero(), big.Zero(), xerrors.Errorf("unsupported actors version")
|
||||
panic("unsupported actors version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,20 +305,13 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch {
|
||||
return miner5.MaxSectorExpirationExtension
|
||||
}
|
||||
|
||||
func GetMinSectorExpiration() abi.ChainEpoch {
|
||||
return miner5.MinSectorExpiration
|
||||
}
|
||||
|
||||
func GetMaxPoStPartitions(nv network.Version, p abi.RegisteredPoStProof) (int, error) {
|
||||
sectorsPerPart, err := builtin5.PoStProofWindowPoStPartitionSectors(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
maxSectors, err := GetAddressedSectorsMax(nv)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(uint64(maxSectors) / sectorsPerPart), nil
|
||||
maxSectors := uint64(GetAddressedSectorsMax(nv))
|
||||
return int(maxSectors / sectorsPerPart), nil
|
||||
}
|
||||
|
||||
func GetDefaultSectorSize() abi.SectorSize {
|
||||
@@ -362,94 +345,82 @@ func GetSectorMaxLifetime(proof abi.RegisteredSealProof, nwVer network.Version)
|
||||
return builtin5.SealProofPoliciesV11[proof].SectorMaxLifetime
|
||||
}
|
||||
|
||||
func GetAddressedSectorsMax(nwVer network.Version) (int, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch v {
|
||||
func GetAddressedSectorsMax(nwVer network.Version) int {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
|
||||
case actors.Version0:
|
||||
return miner0.AddressedSectorsMax, nil
|
||||
return miner0.AddressedSectorsMax
|
||||
|
||||
case actors.Version2:
|
||||
return miner2.AddressedSectorsMax, nil
|
||||
return miner2.AddressedSectorsMax
|
||||
|
||||
case actors.Version3:
|
||||
return miner3.AddressedSectorsMax, nil
|
||||
return miner3.AddressedSectorsMax
|
||||
|
||||
case actors.Version4:
|
||||
return miner4.AddressedSectorsMax, nil
|
||||
return miner4.AddressedSectorsMax
|
||||
|
||||
case actors.Version5:
|
||||
return miner5.AddressedSectorsMax, nil
|
||||
return miner5.AddressedSectorsMax
|
||||
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
func GetDeclarationsMax(nwVer network.Version) (int, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch v {
|
||||
func GetDeclarationsMax(nwVer network.Version) int {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
|
||||
case actors.Version0:
|
||||
|
||||
// TODO: Should we instead error here since the concept doesn't exist yet?
|
||||
return miner0.AddressedPartitionsMax, nil
|
||||
// TODO: Should we instead panic here since the concept doesn't exist yet?
|
||||
return miner0.AddressedPartitionsMax
|
||||
|
||||
case actors.Version2:
|
||||
|
||||
return miner2.DeclarationsMax, nil
|
||||
return miner2.DeclarationsMax
|
||||
|
||||
case actors.Version3:
|
||||
|
||||
return miner3.DeclarationsMax, nil
|
||||
return miner3.DeclarationsMax
|
||||
|
||||
case actors.Version4:
|
||||
|
||||
return miner4.DeclarationsMax, nil
|
||||
return miner4.DeclarationsMax
|
||||
|
||||
case actors.Version5:
|
||||
|
||||
return miner5.DeclarationsMax, nil
|
||||
return miner5.DeclarationsMax
|
||||
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
func AggregateNetworkFee(nwVer network.Version, aggregateSize int, baseFee abi.TokenAmount) (abi.TokenAmount, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return big.Zero(), err
|
||||
}
|
||||
switch v {
|
||||
func AggregateNetworkFee(nwVer network.Version, aggregateSize int, baseFee abi.TokenAmount) abi.TokenAmount {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
|
||||
case actors.Version0:
|
||||
|
||||
return big.Zero(), nil
|
||||
return big.Zero()
|
||||
|
||||
case actors.Version2:
|
||||
|
||||
return big.Zero(), nil
|
||||
return big.Zero()
|
||||
|
||||
case actors.Version3:
|
||||
|
||||
return big.Zero(), nil
|
||||
return big.Zero()
|
||||
|
||||
case actors.Version4:
|
||||
|
||||
return big.Zero(), nil
|
||||
return big.Zero()
|
||||
|
||||
case actors.Version5:
|
||||
|
||||
return miner5.AggregateNetworkFee(aggregateSize, baseFee), nil
|
||||
return miner5.AggregateNetworkFee(aggregateSize, baseFee)
|
||||
|
||||
default:
|
||||
return big.Zero(), xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
@@ -118,18 +117,18 @@ func SetMinVerifiedDealSize(size abi.StoragePower) {
|
||||
{{end}}
|
||||
}
|
||||
|
||||
func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) (abi.ChainEpoch, error) {
|
||||
func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) abi.ChainEpoch {
|
||||
switch ver {
|
||||
{{range .versions}}
|
||||
case actors.Version{{.}}:
|
||||
{{if (eq . 0)}}
|
||||
return miner{{.}}.MaxSealDuration[t], nil
|
||||
return miner{{.}}.MaxSealDuration[t]
|
||||
{{else}}
|
||||
return miner{{.}}.MaxProveCommitDuration[t], nil
|
||||
return miner{{.}}.MaxProveCommitDuration[t]
|
||||
{{end}}
|
||||
{{end}}
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported actors version")
|
||||
panic("unsupported actors version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,24 +150,18 @@ func DealProviderCollateralBounds(
|
||||
size abi.PaddedPieceSize, verified bool,
|
||||
rawBytePower, qaPower, baselinePower abi.StoragePower,
|
||||
circulatingFil abi.TokenAmount, nwVer network.Version,
|
||||
) (min, max abi.TokenAmount, err error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return big.Zero(), big.Zero(), err
|
||||
}
|
||||
switch v {
|
||||
) (min, max abi.TokenAmount) {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
{{range .versions}}
|
||||
case actors.Version{{.}}:
|
||||
{{if (eq . 0)}}
|
||||
min, max := market{{.}}.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil, nwVer)
|
||||
return min, max, nil
|
||||
return market{{.}}.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil, nwVer)
|
||||
{{else}}
|
||||
min, max := market{{.}}.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
return min, max, nil
|
||||
return market{{.}}.DealProviderCollateralBounds(size, verified, rawBytePower, qaPower, baselinePower, circulatingFil)
|
||||
{{end}}
|
||||
{{end}}
|
||||
default:
|
||||
return big.Zero(), big.Zero(), xerrors.Errorf("unsupported actors version")
|
||||
panic("unsupported actors version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,20 +196,13 @@ func GetMaxSectorExpirationExtension() abi.ChainEpoch {
|
||||
return miner{{.latestVersion}}.MaxSectorExpirationExtension
|
||||
}
|
||||
|
||||
func GetMinSectorExpiration() abi.ChainEpoch {
|
||||
return miner{{.latestVersion}}.MinSectorExpiration
|
||||
}
|
||||
|
||||
func GetMaxPoStPartitions(nv network.Version, p abi.RegisteredPoStProof) (int, error) {
|
||||
sectorsPerPart, err := builtin{{.latestVersion}}.PoStProofWindowPoStPartitionSectors(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
maxSectors, err := GetAddressedSectorsMax(nv)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(uint64(maxSectors) / sectorsPerPart), nil
|
||||
maxSectors := uint64(GetAddressedSectorsMax(nv))
|
||||
return int(maxSectors / sectorsPerPart), nil
|
||||
}
|
||||
|
||||
func GetDefaultSectorSize() abi.SectorSize {
|
||||
@@ -250,56 +236,44 @@ func GetSectorMaxLifetime(proof abi.RegisteredSealProof, nwVer network.Version)
|
||||
return builtin{{.latestVersion}}.SealProofPoliciesV11[proof].SectorMaxLifetime
|
||||
}
|
||||
|
||||
func GetAddressedSectorsMax(nwVer network.Version) (int, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch v {
|
||||
func GetAddressedSectorsMax(nwVer network.Version) int {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
{{range .versions}}
|
||||
case actors.Version{{.}}:
|
||||
return miner{{.}}.AddressedSectorsMax, nil
|
||||
return miner{{.}}.AddressedSectorsMax
|
||||
{{end}}
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
func GetDeclarationsMax(nwVer network.Version) (int, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch v {
|
||||
func GetDeclarationsMax(nwVer network.Version) int {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
{{range .versions}}
|
||||
case actors.Version{{.}}:
|
||||
{{if (eq . 0)}}
|
||||
// TODO: Should we instead error here since the concept doesn't exist yet?
|
||||
return miner{{.}}.AddressedPartitionsMax, nil
|
||||
// TODO: Should we instead panic here since the concept doesn't exist yet?
|
||||
return miner{{.}}.AddressedPartitionsMax
|
||||
{{else}}
|
||||
return miner{{.}}.DeclarationsMax, nil
|
||||
return miner{{.}}.DeclarationsMax
|
||||
{{end}}
|
||||
{{end}}
|
||||
default:
|
||||
return 0, xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
func AggregateNetworkFee(nwVer network.Version, aggregateSize int, baseFee abi.TokenAmount) (abi.TokenAmount, error) {
|
||||
v, err := actors.VersionForNetwork(nwVer)
|
||||
if err != nil {
|
||||
return big.Zero(), err
|
||||
}
|
||||
switch v {
|
||||
func AggregateNetworkFee(nwVer network.Version, aggregateSize int, baseFee abi.TokenAmount) abi.TokenAmount {
|
||||
switch actors.VersionForNetwork(nwVer) {
|
||||
{{range .versions}}
|
||||
case actors.Version{{.}}:
|
||||
{{if (le . 4)}}
|
||||
return big.Zero(), nil
|
||||
return big.Zero()
|
||||
{{else}}
|
||||
return miner{{.}}.AggregateNetworkFee(aggregateSize, baseFee), nil
|
||||
return miner{{.}}.AggregateNetworkFee(aggregateSize, baseFee)
|
||||
{{end}}
|
||||
{{end}}
|
||||
default:
|
||||
return big.Zero(), xerrors.Errorf("unsupported network version")
|
||||
panic("unsupported network version")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,19 +21,19 @@ const (
|
||||
)
|
||||
|
||||
// Converts a network version into an actors adt version.
|
||||
func VersionForNetwork(version network.Version) (Version, error) {
|
||||
func VersionForNetwork(version network.Version) Version {
|
||||
switch version {
|
||||
case network.Version0, network.Version1, network.Version2, network.Version3:
|
||||
return Version0, nil
|
||||
return Version0
|
||||
case network.Version4, network.Version5, network.Version6, network.Version7, network.Version8, network.Version9:
|
||||
return Version2, nil
|
||||
return Version2
|
||||
case network.Version10, network.Version11:
|
||||
return Version3, nil
|
||||
return Version3
|
||||
case network.Version12:
|
||||
return Version4, nil
|
||||
return Version4
|
||||
case network.Version13:
|
||||
return Version5, nil
|
||||
return Version5
|
||||
default:
|
||||
return -1, fmt.Errorf("unsupported network version %d", version)
|
||||
panic(fmt.Sprintf("unsupported network version %d", version))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,11 +177,6 @@ func (db *DrandBeacon) VerifyEntry(curr types.BeaconEntry, prev types.BeaconEntr
|
||||
// TODO handle genesis better
|
||||
return nil
|
||||
}
|
||||
|
||||
if curr.Round != prev.Round+1 {
|
||||
return xerrors.Errorf("invalid beacon entry: cur (%d) != prev (%d) + 1", curr.Round, prev.Round)
|
||||
}
|
||||
|
||||
if be := db.getCachedValue(curr.Round); be != nil {
|
||||
if !bytes.Equal(curr.Data, be.Data) {
|
||||
return xerrors.New("invalid beacon value, does not match cached good value")
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
package filcns
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/trace"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
|
||||
exported0 "github.com/filecoin-project/specs-actors/actors/builtin/exported"
|
||||
exported2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/exported"
|
||||
exported3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/exported"
|
||||
exported4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/exported"
|
||||
exported5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/exported"
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/cron"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
)
|
||||
|
||||
func NewActorRegistry() *vm.ActorRegistry {
|
||||
inv := vm.NewActorRegistry()
|
||||
|
||||
// TODO: define all these properties on the actors themselves, in specs-actors.
|
||||
|
||||
inv.Register(vm.ActorsVersionPredicate(actors.Version0), exported0.BuiltinActors()...)
|
||||
inv.Register(vm.ActorsVersionPredicate(actors.Version2), exported2.BuiltinActors()...)
|
||||
inv.Register(vm.ActorsVersionPredicate(actors.Version3), exported3.BuiltinActors()...)
|
||||
inv.Register(vm.ActorsVersionPredicate(actors.Version4), exported4.BuiltinActors()...)
|
||||
inv.Register(vm.ActorsVersionPredicate(actors.Version5), exported5.BuiltinActors()...)
|
||||
|
||||
return inv
|
||||
}
|
||||
|
||||
type TipSetExecutor struct{}
|
||||
|
||||
func NewTipSetExecutor() *TipSetExecutor {
|
||||
return &TipSetExecutor{}
|
||||
}
|
||||
|
||||
func (t *TipSetExecutor) NewActorRegistry() *vm.ActorRegistry {
|
||||
return NewActorRegistry()
|
||||
}
|
||||
|
||||
type FilecoinBlockMessages struct {
|
||||
store.BlockMessages
|
||||
|
||||
WinCount int64
|
||||
}
|
||||
|
||||
func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []FilecoinBlockMessages, epoch abi.ChainEpoch, r vm.Rand, em stmgr.ExecMonitor, baseFee abi.TokenAmount, ts *types.TipSet) (cid.Cid, cid.Cid, error) {
|
||||
done := metrics.Timer(ctx, metrics.VMApplyBlocksTotal)
|
||||
defer done()
|
||||
|
||||
partDone := metrics.Timer(ctx, metrics.VMApplyEarly)
|
||||
defer func() {
|
||||
partDone()
|
||||
}()
|
||||
|
||||
makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) {
|
||||
vmopt := &vm.VMOpts{
|
||||
StateBase: base,
|
||||
Epoch: epoch,
|
||||
Rand: r,
|
||||
Bstore: sm.ChainStore().StateBlockstore(),
|
||||
Actors: NewActorRegistry(),
|
||||
Syscalls: sm.Syscalls,
|
||||
CircSupplyCalc: sm.GetVMCirculatingSupply,
|
||||
NtwkVersion: sm.GetNtwkVersion,
|
||||
BaseFee: baseFee,
|
||||
LookbackState: stmgr.LookbackStateGetterForTipset(sm, ts),
|
||||
}
|
||||
|
||||
return sm.VMConstructor()(ctx, vmopt)
|
||||
}
|
||||
|
||||
vmi, err := makeVmWithBaseState(pstate)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
|
||||
}
|
||||
|
||||
runCron := func(epoch abi.ChainEpoch) error {
|
||||
cronMsg := &types.Message{
|
||||
To: cron.Address,
|
||||
From: builtin.SystemActorAddr,
|
||||
Nonce: uint64(epoch),
|
||||
Value: types.NewInt(0),
|
||||
GasFeeCap: types.NewInt(0),
|
||||
GasPremium: types.NewInt(0),
|
||||
GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little
|
||||
Method: cron.Methods.EpochTick,
|
||||
Params: nil,
|
||||
}
|
||||
ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if em != nil {
|
||||
if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
|
||||
return xerrors.Errorf("callback failed on cron message: %w", err)
|
||||
}
|
||||
}
|
||||
if ret.ExitCode != 0 {
|
||||
return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
for i := parentEpoch; i < epoch; i++ {
|
||||
if i > parentEpoch {
|
||||
// run cron for null rounds if any
|
||||
if err := runCron(i); err != nil {
|
||||
return cid.Undef, cid.Undef, err
|
||||
}
|
||||
|
||||
pstate, err = vmi.Flush(ctx)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handle state forks
|
||||
// XXX: The state tree
|
||||
newState, err := sm.HandleStateForks(ctx, pstate, i, em, ts)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
|
||||
}
|
||||
|
||||
if pstate != newState {
|
||||
vmi, err = makeVmWithBaseState(newState)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
vmi.SetBlockHeight(i + 1)
|
||||
pstate = newState
|
||||
}
|
||||
|
||||
partDone()
|
||||
partDone = metrics.Timer(ctx, metrics.VMApplyMessages)
|
||||
|
||||
var receipts []cbg.CBORMarshaler
|
||||
processedMsgs := make(map[cid.Cid]struct{})
|
||||
for _, b := range bms {
|
||||
penalty := types.NewInt(0)
|
||||
gasReward := big.Zero()
|
||||
|
||||
for _, cm := range append(b.BlsMessages, b.SecpkMessages...) {
|
||||
m := cm.VMMessage()
|
||||
if _, found := processedMsgs[m.Cid()]; found {
|
||||
continue
|
||||
}
|
||||
r, err := vmi.ApplyMessage(ctx, cm)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, err
|
||||
}
|
||||
|
||||
receipts = append(receipts, &r.MessageReceipt)
|
||||
gasReward = big.Add(gasReward, r.GasCosts.MinerTip)
|
||||
penalty = big.Add(penalty, r.GasCosts.MinerPenalty)
|
||||
|
||||
if em != nil {
|
||||
if err := em.MessageApplied(ctx, ts, cm.Cid(), m, r, false); err != nil {
|
||||
return cid.Undef, cid.Undef, err
|
||||
}
|
||||
}
|
||||
processedMsgs[m.Cid()] = struct{}{}
|
||||
}
|
||||
|
||||
params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{
|
||||
Miner: b.Miner,
|
||||
Penalty: penalty,
|
||||
GasReward: gasReward,
|
||||
WinCount: b.WinCount,
|
||||
})
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err)
|
||||
}
|
||||
|
||||
rwMsg := &types.Message{
|
||||
From: builtin.SystemActorAddr,
|
||||
To: reward.Address,
|
||||
Nonce: uint64(epoch),
|
||||
Value: types.NewInt(0),
|
||||
GasFeeCap: types.NewInt(0),
|
||||
GasPremium: types.NewInt(0),
|
||||
GasLimit: 1 << 30,
|
||||
Method: reward.Methods.AwardBlockReward,
|
||||
Params: params,
|
||||
}
|
||||
ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg)
|
||||
if actErr != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("failed to apply reward message for miner %s: %w", b.Miner, actErr)
|
||||
}
|
||||
if em != nil {
|
||||
if err := em.MessageApplied(ctx, ts, rwMsg.Cid(), rwMsg, ret, true); err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on reward message: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if ret.ExitCode != 0 {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
|
||||
}
|
||||
}
|
||||
|
||||
partDone()
|
||||
partDone = metrics.Timer(ctx, metrics.VMApplyCron)
|
||||
|
||||
if err := runCron(epoch); err != nil {
|
||||
return cid.Cid{}, cid.Cid{}, err
|
||||
}
|
||||
|
||||
partDone()
|
||||
partDone = metrics.Timer(ctx, metrics.VMApplyFlush)
|
||||
|
||||
rectarr := blockadt.MakeEmptyArray(sm.ChainStore().ActorStore(ctx))
|
||||
for i, receipt := range receipts {
|
||||
if err := rectarr.Set(uint64(i), receipt); err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
|
||||
}
|
||||
}
|
||||
rectroot, err := rectarr.Root()
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
|
||||
}
|
||||
|
||||
st, err := vmi.Flush(ctx)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("vm flush failed: %w", err)
|
||||
}
|
||||
|
||||
stats.Record(ctx, metrics.VMSends.M(int64(atomic.LoadUint64(&vm.StatSends))),
|
||||
metrics.VMApplied.M(int64(atomic.LoadUint64(&vm.StatApplied))))
|
||||
|
||||
return st, rectroot, nil
|
||||
}
|
||||
|
||||
func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManager, ts *types.TipSet, em stmgr.ExecMonitor) (stateroot cid.Cid, rectsroot cid.Cid, err error) {
|
||||
ctx, span := trace.StartSpan(ctx, "computeTipSetState")
|
||||
defer span.End()
|
||||
|
||||
blks := ts.Blocks()
|
||||
|
||||
for i := 0; i < len(blks); i++ {
|
||||
for j := i + 1; j < len(blks); j++ {
|
||||
if blks[i].Miner == blks[j].Miner {
|
||||
return cid.Undef, cid.Undef,
|
||||
xerrors.Errorf("duplicate miner in a tipset (%s %s)",
|
||||
blks[i].Miner, blks[j].Miner)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parentEpoch abi.ChainEpoch
|
||||
pstate := blks[0].ParentStateRoot
|
||||
if blks[0].Height > 0 {
|
||||
parent, err := sm.ChainStore().GetBlock(blks[0].Parents[0])
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
|
||||
}
|
||||
|
||||
parentEpoch = parent.Height
|
||||
}
|
||||
|
||||
r := store.NewChainRand(sm.ChainStore(), ts.Cids())
|
||||
|
||||
blkmsgs, err := sm.ChainStore().BlockMsgsForTipset(ts)
|
||||
if err != nil {
|
||||
return cid.Undef, cid.Undef, xerrors.Errorf("getting block messages for tipset: %w", err)
|
||||
}
|
||||
fbmsgs := make([]FilecoinBlockMessages, len(blkmsgs))
|
||||
for i := range fbmsgs {
|
||||
fbmsgs[i].BlockMessages = blkmsgs[i]
|
||||
fbmsgs[i].WinCount = ts.Blocks()[i].ElectionProof.WinCount
|
||||
}
|
||||
baseFee := blks[0].ParentBaseFee
|
||||
|
||||
return t.ApplyBlocks(ctx, sm, parentEpoch, pstate, fbmsgs, blks[0].Height, r, em, baseFee, ts)
|
||||
}
|
||||
|
||||
var _ stmgr.Executor = &TipSetExecutor{}
|
||||
@@ -1,847 +0,0 @@
|
||||
package filcns
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/ipfs/go-cid"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"go.opencensus.io/stats"
|
||||
"go.opencensus.io/trace"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
|
||||
"github.com/filecoin-project/lotus/chain/beacon"
|
||||
"github.com/filecoin-project/lotus/chain/consensus"
|
||||
"github.com/filecoin-project/lotus/chain/state"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/vm"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
|
||||
"github.com/filecoin-project/lotus/lib/async"
|
||||
"github.com/filecoin-project/lotus/lib/sigs"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
)
|
||||
|
||||
var log = logging.Logger("fil-consensus")
|
||||
|
||||
type FilecoinEC struct {
|
||||
// The interface for accessing and putting tipsets into local storage
|
||||
store *store.ChainStore
|
||||
|
||||
// handle to the random beacon for verification
|
||||
beacon beacon.Schedule
|
||||
|
||||
// the state manager handles making state queries
|
||||
sm *stmgr.StateManager
|
||||
|
||||
verifier ffiwrapper.Verifier
|
||||
|
||||
genesis *types.TipSet
|
||||
}
|
||||
|
||||
// Blocks that are more than MaxHeightDrift epochs above
|
||||
// the theoretical max height based on systime are quickly rejected
|
||||
const MaxHeightDrift = 5
|
||||
|
||||
func NewFilecoinExpectedConsensus(sm *stmgr.StateManager, beacon beacon.Schedule, verifier ffiwrapper.Verifier, genesis chain.Genesis) consensus.Consensus {
|
||||
if build.InsecurePoStValidation {
|
||||
log.Warn("*********************************************************************************************")
|
||||
log.Warn(" [INSECURE-POST-VALIDATION] Insecure test validation is enabled. If you see this outside of a test, it is a severe bug! ")
|
||||
log.Warn("*********************************************************************************************")
|
||||
}
|
||||
|
||||
return &FilecoinEC{
|
||||
store: sm.ChainStore(),
|
||||
beacon: beacon,
|
||||
sm: sm,
|
||||
verifier: verifier,
|
||||
genesis: genesis,
|
||||
}
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) {
|
||||
if err := blockSanityChecks(b.Header); err != nil {
|
||||
return xerrors.Errorf("incoming header failed basic sanity checks: %w", err)
|
||||
}
|
||||
|
||||
h := b.Header
|
||||
|
||||
baseTs, err := filec.store.LoadTipSet(types.NewTipSetKey(h.Parents...))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("load parent tipset failed (%s): %w", h.Parents, err)
|
||||
}
|
||||
|
||||
winPoStNv := filec.sm.GetNtwkVersion(ctx, baseTs.Height())
|
||||
|
||||
lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, filec.sm, baseTs, h.Height)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get lookback tipset for block: %w", err)
|
||||
}
|
||||
|
||||
prevBeacon, err := filec.store.GetLatestBeaconEntry(baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get latest beacon entry: %w", err)
|
||||
}
|
||||
|
||||
// fast checks first
|
||||
if h.Height <= baseTs.Height() {
|
||||
return xerrors.Errorf("block height not greater than parent height: %d != %d", h.Height, baseTs.Height())
|
||||
}
|
||||
|
||||
nulls := h.Height - (baseTs.Height() + 1)
|
||||
if tgtTs := baseTs.MinTimestamp() + build.BlockDelaySecs*uint64(nulls+1); h.Timestamp != tgtTs {
|
||||
return xerrors.Errorf("block has wrong timestamp: %d != %d", h.Timestamp, tgtTs)
|
||||
}
|
||||
|
||||
now := uint64(build.Clock.Now().Unix())
|
||||
if h.Timestamp > now+build.AllowableClockDriftSecs {
|
||||
return xerrors.Errorf("block was from the future (now=%d, blk=%d): %w", now, h.Timestamp, consensus.ErrTemporal)
|
||||
}
|
||||
if h.Timestamp > now {
|
||||
log.Warn("Got block from the future, but within threshold", h.Timestamp, build.Clock.Now().Unix())
|
||||
}
|
||||
|
||||
msgsCheck := async.Err(func() error {
|
||||
if b.Cid() == build.WhitelistedBlock {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := filec.checkBlockMessages(ctx, b, baseTs); err != nil {
|
||||
return xerrors.Errorf("block had invalid messages: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
minerCheck := async.Err(func() error {
|
||||
if err := filec.minerIsValid(ctx, h.Miner, baseTs); err != nil {
|
||||
return xerrors.Errorf("minerIsValid failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
baseFeeCheck := async.Err(func() error {
|
||||
baseFee, err := filec.store.ComputeBaseFee(ctx, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("computing base fee: %w", err)
|
||||
}
|
||||
if types.BigCmp(baseFee, b.Header.ParentBaseFee) != 0 {
|
||||
return xerrors.Errorf("base fee doesn't match: %s (header) != %s (computed)",
|
||||
b.Header.ParentBaseFee, baseFee)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
pweight, err := filec.store.Weight(ctx, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting parent weight: %w", err)
|
||||
}
|
||||
|
||||
if types.BigCmp(pweight, b.Header.ParentWeight) != 0 {
|
||||
return xerrors.Errorf("parrent weight different: %s (header) != %s (computed)",
|
||||
b.Header.ParentWeight, pweight)
|
||||
}
|
||||
|
||||
stateRootCheck := async.Err(func() error {
|
||||
stateroot, precp, err := filec.sm.TipSetState(ctx, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err)
|
||||
}
|
||||
|
||||
if stateroot != h.ParentStateRoot {
|
||||
msgs, err := filec.store.MessagesForTipset(baseTs)
|
||||
if err != nil {
|
||||
log.Error("failed to load messages for tipset during tipset state mismatch error: ", err)
|
||||
} else {
|
||||
log.Warn("Messages for tipset with mismatching state:")
|
||||
for i, m := range msgs {
|
||||
mm := m.VMMessage()
|
||||
log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params)
|
||||
}
|
||||
}
|
||||
|
||||
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
|
||||
}
|
||||
|
||||
if precp != h.ParentMessageReceipts {
|
||||
return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
// Stuff that needs worker address
|
||||
waddr, err := stmgr.GetMinerWorkerRaw(ctx, filec.sm, lbst, h.Miner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err)
|
||||
}
|
||||
|
||||
winnerCheck := async.Err(func() error {
|
||||
if h.ElectionProof.WinCount < 1 {
|
||||
return xerrors.Errorf("block is not claiming to be a winner")
|
||||
}
|
||||
|
||||
eligible, err := stmgr.MinerEligibleToMine(ctx, filec.sm, h.Miner, baseTs, lbts)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("determining if miner has min power failed: %w", err)
|
||||
}
|
||||
|
||||
if !eligible {
|
||||
return xerrors.New("block's miner is ineligible to mine")
|
||||
}
|
||||
|
||||
rBeacon := *prevBeacon
|
||||
if len(h.BeaconEntries) != 0 {
|
||||
rBeacon = h.BeaconEntries[len(h.BeaconEntries)-1]
|
||||
}
|
||||
buf := new(bytes.Buffer)
|
||||
if err := h.Miner.MarshalCBOR(buf); err != nil {
|
||||
return xerrors.Errorf("failed to marshal miner address to cbor: %w", err)
|
||||
}
|
||||
|
||||
vrfBase, err := store.DrawRandomness(rBeacon.Data, crypto.DomainSeparationTag_ElectionProofProduction, h.Height, buf.Bytes())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("could not draw randomness: %w", err)
|
||||
}
|
||||
|
||||
if err := VerifyElectionPoStVRF(ctx, waddr, vrfBase, h.ElectionProof.VRFProof); err != nil {
|
||||
return xerrors.Errorf("validating block election proof failed: %w", err)
|
||||
}
|
||||
|
||||
slashed, err := stmgr.GetMinerSlashed(ctx, filec.sm, baseTs, h.Miner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to check if block miner was slashed: %w", err)
|
||||
}
|
||||
|
||||
if slashed {
|
||||
return xerrors.Errorf("received block was from slashed or invalid miner")
|
||||
}
|
||||
|
||||
mpow, tpow, _, err := stmgr.GetPowerRaw(ctx, filec.sm, lbst, h.Miner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed getting power: %w", err)
|
||||
}
|
||||
|
||||
j := h.ElectionProof.ComputeWinCount(mpow.QualityAdjPower, tpow.QualityAdjPower)
|
||||
if h.ElectionProof.WinCount != j {
|
||||
return xerrors.Errorf("miner claims wrong number of wins: miner: %d, computed: %d", h.ElectionProof.WinCount, j)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
blockSigCheck := async.Err(func() error {
|
||||
if err := sigs.CheckBlockSignature(ctx, h, waddr); err != nil {
|
||||
return xerrors.Errorf("check block signature failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
beaconValuesCheck := async.Err(func() error {
|
||||
if os.Getenv("LOTUS_IGNORE_DRAND") == "_yes_" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := beacon.ValidateBlockValues(filec.beacon, h, baseTs.Height(), *prevBeacon); err != nil {
|
||||
return xerrors.Errorf("failed to validate blocks random beacon values: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
tktsCheck := async.Err(func() error {
|
||||
buf := new(bytes.Buffer)
|
||||
if err := h.Miner.MarshalCBOR(buf); err != nil {
|
||||
return xerrors.Errorf("failed to marshal miner address to cbor: %w", err)
|
||||
}
|
||||
|
||||
if h.Height > build.UpgradeSmokeHeight {
|
||||
buf.Write(baseTs.MinTicket().VRFProof)
|
||||
}
|
||||
|
||||
beaconBase := *prevBeacon
|
||||
if len(h.BeaconEntries) != 0 {
|
||||
beaconBase = h.BeaconEntries[len(h.BeaconEntries)-1]
|
||||
}
|
||||
|
||||
vrfBase, err := store.DrawRandomness(beaconBase.Data, crypto.DomainSeparationTag_TicketProduction, h.Height-build.TicketRandomnessLookback, buf.Bytes())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to compute vrf base for ticket: %w", err)
|
||||
}
|
||||
|
||||
err = VerifyElectionPoStVRF(ctx, waddr, vrfBase, h.Ticket.VRFProof)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("validating block tickets failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
wproofCheck := async.Err(func() error {
|
||||
if err := filec.VerifyWinningPoStProof(ctx, winPoStNv, h, *prevBeacon, lbst, waddr); err != nil {
|
||||
return xerrors.Errorf("invalid election post: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
await := []async.ErrorFuture{
|
||||
minerCheck,
|
||||
tktsCheck,
|
||||
blockSigCheck,
|
||||
beaconValuesCheck,
|
||||
wproofCheck,
|
||||
winnerCheck,
|
||||
msgsCheck,
|
||||
baseFeeCheck,
|
||||
stateRootCheck,
|
||||
}
|
||||
|
||||
var merr error
|
||||
for _, fut := range await {
|
||||
if err := fut.AwaitContext(ctx); err != nil {
|
||||
merr = multierror.Append(merr, err)
|
||||
}
|
||||
}
|
||||
if merr != nil {
|
||||
mulErr := merr.(*multierror.Error)
|
||||
mulErr.ErrorFormat = func(es []error) string {
|
||||
if len(es) == 1 {
|
||||
return fmt.Sprintf("1 error occurred:\n\t* %+v\n\n", es[0])
|
||||
}
|
||||
|
||||
points := make([]string, len(es))
|
||||
for i, err := range es {
|
||||
points[i] = fmt.Sprintf("* %+v", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"%d errors occurred:\n\t%s\n\n",
|
||||
len(es), strings.Join(points, "\n\t"))
|
||||
}
|
||||
return mulErr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func blockSanityChecks(h *types.BlockHeader) error {
|
||||
if h.ElectionProof == nil {
|
||||
return xerrors.Errorf("block cannot have nil election proof")
|
||||
}
|
||||
|
||||
if h.Ticket == nil {
|
||||
return xerrors.Errorf("block cannot have nil ticket")
|
||||
}
|
||||
|
||||
if h.BlockSig == nil {
|
||||
return xerrors.Errorf("block had nil signature")
|
||||
}
|
||||
|
||||
if h.BLSAggregate == nil {
|
||||
return xerrors.Errorf("block had nil bls aggregate signature")
|
||||
}
|
||||
|
||||
if h.Miner.Protocol() != address.ID {
|
||||
return xerrors.Errorf("block had non-ID miner address")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) VerifyWinningPoStProof(ctx context.Context, nv network.Version, h *types.BlockHeader, prevBeacon types.BeaconEntry, lbst cid.Cid, waddr address.Address) error {
|
||||
if build.InsecurePoStValidation {
|
||||
if len(h.WinPoStProof) == 0 {
|
||||
return xerrors.Errorf("[INSECURE-POST-VALIDATION] No winning post proof given")
|
||||
}
|
||||
|
||||
if string(h.WinPoStProof[0].ProofBytes) == "valid proof" {
|
||||
return nil
|
||||
}
|
||||
return xerrors.Errorf("[INSECURE-POST-VALIDATION] winning post was invalid")
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if err := h.Miner.MarshalCBOR(buf); err != nil {
|
||||
return xerrors.Errorf("failed to marshal miner address: %w", err)
|
||||
}
|
||||
|
||||
rbase := prevBeacon
|
||||
if len(h.BeaconEntries) > 0 {
|
||||
rbase = h.BeaconEntries[len(h.BeaconEntries)-1]
|
||||
}
|
||||
|
||||
rand, err := store.DrawRandomness(rbase.Data, crypto.DomainSeparationTag_WinningPoStChallengeSeed, h.Height, buf.Bytes())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get randomness for verifying winning post proof: %w", err)
|
||||
}
|
||||
|
||||
mid, err := address.IDFromAddress(h.Miner)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get ID from miner address %s: %w", h.Miner, err)
|
||||
}
|
||||
|
||||
sectors, err := stmgr.GetSectorsForWinningPoSt(ctx, nv, filec.verifier, filec.sm, lbst, h.Miner, rand)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("getting winning post sector set: %w", err)
|
||||
}
|
||||
|
||||
ok, err := ffiwrapper.ProofVerifier.VerifyWinningPoSt(ctx, proof2.WinningPoStVerifyInfo{
|
||||
Randomness: rand,
|
||||
Proofs: h.WinPoStProof,
|
||||
ChallengedSectors: sectors,
|
||||
Prover: abi.ActorID(mid),
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to verify election post: %w", err)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
log.Errorf("invalid winning post (block: %s, %x; %v)", h.Cid(), rand, sectors)
|
||||
return xerrors.Errorf("winning post was invalid")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: We should extract this somewhere else and make the message pool and miner use the same logic
|
||||
func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBlock, baseTs *types.TipSet) error {
|
||||
{
|
||||
var sigCids []cid.Cid // this is what we get for people not wanting the marshalcbor method on the cid type
|
||||
var pubks [][]byte
|
||||
|
||||
for _, m := range b.BlsMessages {
|
||||
sigCids = append(sigCids, m.Cid())
|
||||
|
||||
pubk, err := filec.sm.GetBlsPublicKey(ctx, m.From, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load bls public to validate block: %w", err)
|
||||
}
|
||||
|
||||
pubks = append(pubks, pubk)
|
||||
}
|
||||
|
||||
if err := consensus.VerifyBlsAggregate(ctx, b.Header.BLSAggregate, sigCids, pubks); err != nil {
|
||||
return xerrors.Errorf("bls aggregate signature was invalid: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
nonces := make(map[address.Address]uint64)
|
||||
|
||||
stateroot, _, err := filec.sm.TipSetState(ctx, baseTs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
st, err := state.LoadStateTree(filec.store.ActorStore(ctx), stateroot)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load base state tree: %w", err)
|
||||
}
|
||||
|
||||
nv := filec.sm.GetNtwkVersion(ctx, b.Header.Height)
|
||||
pl := vm.PricelistByEpoch(baseTs.Height())
|
||||
var sumGasLimit int64
|
||||
checkMsg := func(msg types.ChainMsg) error {
|
||||
m := msg.VMMessage()
|
||||
|
||||
// Phase 1: syntactic validation, as defined in the spec
|
||||
minGas := pl.OnChainMessage(msg.ChainLength())
|
||||
if err := m.ValidForBlockInclusion(minGas.Total(), nv); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ValidForBlockInclusion checks if any single message does not exceed BlockGasLimit
|
||||
// So below is overflow safe
|
||||
sumGasLimit += m.GasLimit
|
||||
if sumGasLimit > build.BlockGasLimit {
|
||||
return xerrors.Errorf("block gas limit exceeded")
|
||||
}
|
||||
|
||||
// Phase 2: (Partial) semantic validation:
|
||||
// the sender exists and is an account actor, and the nonces make sense
|
||||
var sender address.Address
|
||||
if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version13 {
|
||||
sender, err = st.LookupID(m.From)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
sender = m.From
|
||||
}
|
||||
|
||||
if _, ok := nonces[sender]; !ok {
|
||||
// `GetActor` does not validate that this is an account actor.
|
||||
act, err := st.GetActor(sender)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get actor: %w", err)
|
||||
}
|
||||
|
||||
if !builtin.IsAccountActor(act.Code) {
|
||||
return xerrors.New("Sender must be an account actor")
|
||||
}
|
||||
nonces[sender] = act.Nonce
|
||||
}
|
||||
|
||||
if nonces[sender] != m.Nonce {
|
||||
return xerrors.Errorf("wrong nonce (exp: %d, got: %d)", nonces[sender], m.Nonce)
|
||||
}
|
||||
nonces[sender]++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate message arrays in a temporary blockstore.
|
||||
tmpbs := bstore.NewMemory()
|
||||
tmpstore := blockadt.WrapStore(ctx, cbor.NewCborStore(tmpbs))
|
||||
|
||||
bmArr := blockadt.MakeEmptyArray(tmpstore)
|
||||
for i, m := range b.BlsMessages {
|
||||
if err := checkMsg(m); err != nil {
|
||||
return xerrors.Errorf("block had invalid bls message at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
c, err := store.PutMessage(tmpbs, m)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err)
|
||||
}
|
||||
|
||||
k := cbg.CborCid(c)
|
||||
if err := bmArr.Set(uint64(i), &k); err != nil {
|
||||
return xerrors.Errorf("failed to put bls message at index %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
smArr := blockadt.MakeEmptyArray(tmpstore)
|
||||
for i, m := range b.SecpkMessages {
|
||||
if err := checkMsg(m); err != nil {
|
||||
return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
// `From` being an account actor is only validated inside the `vm.ResolveToKeyAddr` call
|
||||
// in `StateManager.ResolveToKeyAddress` here (and not in `checkMsg`).
|
||||
kaddr, err := filec.sm.ResolveToKeyAddress(ctx, m.Message.From, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to resolve key addr: %w", err)
|
||||
}
|
||||
|
||||
if err := sigs.Verify(&m.Signature, kaddr, m.Message.Cid().Bytes()); err != nil {
|
||||
return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err)
|
||||
}
|
||||
|
||||
c, err := store.PutMessage(tmpbs, m)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to store message %s: %w", m.Cid(), err)
|
||||
}
|
||||
k := cbg.CborCid(c)
|
||||
if err := smArr.Set(uint64(i), &k); err != nil {
|
||||
return xerrors.Errorf("failed to put secpk message at index %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
bmroot, err := bmArr.Root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
smroot, err := smArr.Root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{
|
||||
BlsMessages: bmroot,
|
||||
SecpkMessages: smroot,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if b.Header.Messages != mrcid {
|
||||
return fmt.Errorf("messages didnt match message root in header")
|
||||
}
|
||||
|
||||
// Finally, flush.
|
||||
return vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
|
||||
if filec.genesis == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
now := uint64(build.Clock.Now().Unix())
|
||||
return epoch > (abi.ChainEpoch((now-filec.genesis.MinTimestamp())/build.BlockDelaySecs) + MaxHeightDrift)
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) minerIsValid(ctx context.Context, maddr address.Address, baseTs *types.TipSet) error {
|
||||
act, err := filec.sm.LoadActor(ctx, power.Address, baseTs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load power actor: %w", err)
|
||||
}
|
||||
|
||||
powState, err := power.Load(filec.store.ActorStore(ctx), act)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to load power actor state: %w", err)
|
||||
}
|
||||
|
||||
_, exist, err := powState.MinerPower(maddr)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to look up miner's claim: %w", err)
|
||||
}
|
||||
|
||||
if !exist {
|
||||
return xerrors.New("miner isn't valid")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func VerifyElectionPoStVRF(ctx context.Context, worker address.Address, rand []byte, evrf []byte) error {
|
||||
return VerifyVRF(ctx, worker, rand, evrf)
|
||||
}
|
||||
|
||||
func VerifyVRF(ctx context.Context, worker address.Address, vrfBase, vrfproof []byte) error {
|
||||
_, span := trace.StartSpan(ctx, "VerifyVRF")
|
||||
defer span.End()
|
||||
|
||||
sig := &crypto.Signature{
|
||||
Type: crypto.SigTypeBLS,
|
||||
Data: vrfproof,
|
||||
}
|
||||
|
||||
if err := sigs.Verify(sig, worker, vrfBase); err != nil {
|
||||
return xerrors.Errorf("vrf was invalid: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrSoftFailure = errors.New("soft validation failure")
|
||||
var ErrInsufficientPower = errors.New("incoming block's miner does not have minimum power")
|
||||
|
||||
func (filec *FilecoinEC) ValidateBlockPubsub(ctx context.Context, self bool, msg *pubsub.Message) (pubsub.ValidationResult, string) {
|
||||
if self {
|
||||
return filec.validateLocalBlock(ctx, msg)
|
||||
}
|
||||
|
||||
// track validation time
|
||||
begin := build.Clock.Now()
|
||||
defer func() {
|
||||
log.Debugf("block validation time: %s", build.Clock.Since(begin))
|
||||
}()
|
||||
|
||||
stats.Record(ctx, metrics.BlockReceived.M(1))
|
||||
|
||||
recordFailureFlagPeer := func(what string) {
|
||||
// bv.Validate will flag the peer in that case
|
||||
panic(what)
|
||||
}
|
||||
|
||||
blk, what, err := filec.decodeAndCheckBlock(msg)
|
||||
if err != nil {
|
||||
log.Error("got invalid block over pubsub: ", err)
|
||||
recordFailureFlagPeer(what)
|
||||
return pubsub.ValidationReject, what
|
||||
}
|
||||
|
||||
// validate the block meta: the Message CID in the header must match the included messages
|
||||
err = filec.validateMsgMeta(ctx, blk)
|
||||
if err != nil {
|
||||
log.Warnf("error validating message metadata: %s", err)
|
||||
recordFailureFlagPeer("invalid_block_meta")
|
||||
return pubsub.ValidationReject, "invalid_block_meta"
|
||||
}
|
||||
|
||||
reject, err := filec.validateBlockHeader(ctx, blk.Header)
|
||||
if err != nil {
|
||||
if reject == "" {
|
||||
log.Warn("ignoring block msg: ", err)
|
||||
return pubsub.ValidationIgnore, reject
|
||||
}
|
||||
recordFailureFlagPeer(reject)
|
||||
return pubsub.ValidationReject, reject
|
||||
}
|
||||
|
||||
// all good, accept the block
|
||||
msg.ValidatorData = blk
|
||||
stats.Record(ctx, metrics.BlockValidationSuccess.M(1))
|
||||
return pubsub.ValidationAccept, ""
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) validateLocalBlock(ctx context.Context, msg *pubsub.Message) (pubsub.ValidationResult, string) {
|
||||
stats.Record(ctx, metrics.BlockPublished.M(1))
|
||||
|
||||
if size := msg.Size(); size > 1<<20-1<<15 {
|
||||
log.Errorf("ignoring oversize block (%dB)", size)
|
||||
return pubsub.ValidationIgnore, "oversize_block"
|
||||
}
|
||||
|
||||
blk, what, err := filec.decodeAndCheckBlock(msg)
|
||||
if err != nil {
|
||||
log.Errorf("got invalid local block: %s", err)
|
||||
return pubsub.ValidationIgnore, what
|
||||
}
|
||||
|
||||
msg.ValidatorData = blk
|
||||
stats.Record(ctx, metrics.BlockValidationSuccess.M(1))
|
||||
return pubsub.ValidationAccept, ""
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) decodeAndCheckBlock(msg *pubsub.Message) (*types.BlockMsg, string, error) {
|
||||
blk, err := types.DecodeBlockMsg(msg.GetData())
|
||||
if err != nil {
|
||||
return nil, "invalid", xerrors.Errorf("error decoding block: %w", err)
|
||||
}
|
||||
|
||||
if count := len(blk.BlsMessages) + len(blk.SecpkMessages); count > build.BlockMessageLimit {
|
||||
return nil, "too_many_messages", fmt.Errorf("block contains too many messages (%d)", count)
|
||||
}
|
||||
|
||||
// make sure we have a signature
|
||||
if blk.Header.BlockSig == nil {
|
||||
return nil, "missing_signature", fmt.Errorf("block without a signature")
|
||||
}
|
||||
|
||||
return blk, "", nil
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) validateMsgMeta(ctx context.Context, msg *types.BlockMsg) error {
|
||||
// TODO there has to be a simpler way to do this without the blockstore dance
|
||||
// block headers use adt0
|
||||
store := blockadt.WrapStore(ctx, cbor.NewCborStore(bstore.NewMemory()))
|
||||
bmArr := blockadt.MakeEmptyArray(store)
|
||||
smArr := blockadt.MakeEmptyArray(store)
|
||||
|
||||
for i, m := range msg.BlsMessages {
|
||||
c := cbg.CborCid(m)
|
||||
if err := bmArr.Set(uint64(i), &c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for i, m := range msg.SecpkMessages {
|
||||
c := cbg.CborCid(m)
|
||||
if err := smArr.Set(uint64(i), &c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
bmroot, err := bmArr.Root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
smroot, err := smArr.Root()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mrcid, err := store.Put(store.Context(), &types.MsgMeta{
|
||||
BlsMessages: bmroot,
|
||||
SecpkMessages: smroot,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if msg.Header.Messages != mrcid {
|
||||
return fmt.Errorf("messages didn't match root cid in header")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) validateBlockHeader(ctx context.Context, b *types.BlockHeader) (rejectReason string, err error) {
|
||||
|
||||
// we want to ensure that it is a block from a known miner; we reject blocks from unknown miners
|
||||
// to prevent spam attacks.
|
||||
// the logic works as follows: we lookup the miner in the chain for its key.
|
||||
// if we can find it then it's a known miner and we can validate the signature.
|
||||
// if we can't find it, we check whether we are (near) synced in the chain.
|
||||
// if we are not synced we cannot validate the block and we must ignore it.
|
||||
// if we are synced and the miner is unknown, then the block is rejcected.
|
||||
key, err := filec.checkPowerAndGetWorkerKey(ctx, b)
|
||||
if err != nil {
|
||||
if err != ErrSoftFailure && filec.isChainNearSynced() {
|
||||
log.Warnf("received block from unknown miner or miner that doesn't meet min power over pubsub; rejecting message")
|
||||
return "unknown_miner", err
|
||||
}
|
||||
|
||||
log.Warnf("cannot validate block message; unknown miner or miner that doesn't meet min power in unsynced chain: %s", b.Cid())
|
||||
return "", err // ignore
|
||||
}
|
||||
|
||||
if b.ElectionProof.WinCount < 1 {
|
||||
log.Errorf("block is not claiming to be winning")
|
||||
return "not_winning", xerrors.Errorf("block not winning")
|
||||
}
|
||||
|
||||
err = sigs.CheckBlockSignature(ctx, b, key)
|
||||
if err != nil {
|
||||
log.Errorf("block signature verification failed: %s", err)
|
||||
return "signature_verification_failed", err
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) checkPowerAndGetWorkerKey(ctx context.Context, bh *types.BlockHeader) (address.Address, error) {
|
||||
// we check that the miner met the minimum power at the lookback tipset
|
||||
|
||||
baseTs := filec.store.GetHeaviestTipSet()
|
||||
lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, filec.sm, baseTs, bh.Height)
|
||||
if err != nil {
|
||||
log.Warnf("failed to load lookback tipset for incoming block: %s", err)
|
||||
return address.Undef, ErrSoftFailure
|
||||
}
|
||||
|
||||
key, err := stmgr.GetMinerWorkerRaw(ctx, filec.sm, lbst, bh.Miner)
|
||||
if err != nil {
|
||||
log.Warnf("failed to resolve worker key for miner %s: %s", bh.Miner, err)
|
||||
return address.Undef, ErrSoftFailure
|
||||
}
|
||||
|
||||
// NOTE: we check to see if the miner was eligible in the lookback
|
||||
// tipset - 1 for historical reasons. DO NOT use the lookback state
|
||||
// returned by GetLookbackTipSetForRound.
|
||||
|
||||
eligible, err := stmgr.MinerEligibleToMine(ctx, filec.sm, bh.Miner, baseTs, lbts)
|
||||
if err != nil {
|
||||
log.Warnf("failed to determine if incoming block's miner has minimum power: %s", err)
|
||||
return address.Undef, ErrSoftFailure
|
||||
}
|
||||
|
||||
if !eligible {
|
||||
log.Warnf("incoming block's miner is ineligible")
|
||||
return address.Undef, ErrInsufficientPower
|
||||
}
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (filec *FilecoinEC) isChainNearSynced() bool {
|
||||
ts := filec.store.GetHeaviestTipSet()
|
||||
timestamp := ts.MinTimestamp()
|
||||
timestampTime := time.Unix(int64(timestamp), 0)
|
||||
return build.Clock.Since(timestampTime) < 6*time.Hour
|
||||
}
|
||||
|
||||
var _ consensus.Consensus = &FilecoinEC{}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,19 +0,0 @@
|
||||
package consensus
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
type Consensus interface {
|
||||
ValidateBlock(ctx context.Context, b *types.FullBlock) (err error)
|
||||
ValidateBlockPubsub(ctx context.Context, self bool, msg *pubsub.Message) (pubsub.ValidationResult, string)
|
||||
IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool
|
||||
|
||||
CreateBlock(ctx context.Context, w api.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error)
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package consensus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
|
||||
"github.com/ipfs/go-cid"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"go.opencensus.io/trace"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
)
|
||||
|
||||
var ErrTemporal = errors.New("temporal error")
|
||||
|
||||
func VerifyBlsAggregate(ctx context.Context, sig *crypto.Signature, msgs []cid.Cid, pubks [][]byte) error {
|
||||
_, span := trace.StartSpan(ctx, "syncer.VerifyBlsAggregate")
|
||||
defer span.End()
|
||||
span.AddAttributes(
|
||||
trace.Int64Attribute("msgCount", int64(len(msgs))),
|
||||
)
|
||||
|
||||
msgsS := make([]ffi.Message, len(msgs))
|
||||
pubksS := make([]ffi.PublicKey, len(msgs))
|
||||
for i := 0; i < len(msgs); i++ {
|
||||
msgsS[i] = msgs[i].Bytes()
|
||||
copy(pubksS[i][:], pubks[i][:ffi.PublicKeyBytes])
|
||||
}
|
||||
|
||||
sigS := new(ffi.Signature)
|
||||
copy(sigS[:], sig.Data[:ffi.SignatureBytes])
|
||||
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
valid := ffi.HashVerify(sigS, msgsS, pubksS)
|
||||
if !valid {
|
||||
return xerrors.New("bls aggregate signature failed to verify")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func AggregateSignatures(sigs []crypto.Signature) (*crypto.Signature, error) {
|
||||
sigsS := make([]ffi.Signature, len(sigs))
|
||||
for i := 0; i < len(sigs); i++ {
|
||||
copy(sigsS[i][:], sigs[i].Data[:ffi.SignatureBytes])
|
||||
}
|
||||
|
||||
aggSig := ffi.Aggregate(sigsS)
|
||||
if aggSig == nil {
|
||||
if len(sigs) > 0 {
|
||||
return nil, xerrors.Errorf("bls.Aggregate returned nil with %d signatures", len(sigs))
|
||||
}
|
||||
|
||||
zeroSig := ffi.CreateZeroSignature()
|
||||
|
||||
// Note: for blst this condition should not happen - nil should not
|
||||
// be returned
|
||||
return &crypto.Signature{
|
||||
Type: crypto.SigTypeBLS,
|
||||
Data: zeroSig[:],
|
||||
}, nil
|
||||
}
|
||||
return &crypto.Signature{
|
||||
Type: crypto.SigTypeBLS,
|
||||
Data: aggSig[:],
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ToMessagesArray(store blockadt.Store, cids []cid.Cid) (cid.Cid, error) {
|
||||
arr := blockadt.MakeEmptyArray(store)
|
||||
for i, c := range cids {
|
||||
oc := cbg.CborCid(c)
|
||||
if err := arr.Set(uint64(i), &oc); err != nil {
|
||||
return cid.Undef, err
|
||||
}
|
||||
}
|
||||
return arr.Root()
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type uncachedAPI interface {
|
||||
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
|
||||
ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error)
|
||||
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error)
|
||||
|
||||
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg
|
||||
}
|
||||
|
||||
type cache struct {
|
||||
*tipSetCache
|
||||
*messageCache
|
||||
uncachedAPI
|
||||
}
|
||||
|
||||
func newCache(api EventAPI, gcConfidence abi.ChainEpoch) *cache {
|
||||
return &cache{
|
||||
newTSCache(api, gcConfidence),
|
||||
newMessageCache(api),
|
||||
api,
|
||||
}
|
||||
}
|
||||
+184
-17
@@ -2,14 +2,18 @@ package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/ipfs/go-cid"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
@@ -21,46 +25,209 @@ type (
|
||||
RevertHandler func(ctx context.Context, ts *types.TipSet) error
|
||||
)
|
||||
|
||||
// A TipSetObserver receives notifications of tipsets
|
||||
type TipSetObserver interface {
|
||||
Apply(ctx context.Context, from, to *types.TipSet) error
|
||||
Revert(ctx context.Context, from, to *types.TipSet) error
|
||||
type heightHandler struct {
|
||||
confidence int
|
||||
called bool
|
||||
|
||||
handle HeightHandler
|
||||
revert RevertHandler
|
||||
}
|
||||
|
||||
type EventAPI interface {
|
||||
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
|
||||
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
|
||||
ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainHead(context.Context) (*types.TipSet, error)
|
||||
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error)
|
||||
ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error)
|
||||
|
||||
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) // optional / for CalledMsg
|
||||
}
|
||||
|
||||
type Events struct {
|
||||
*observer
|
||||
*heightEvents
|
||||
api EventAPI
|
||||
|
||||
tsc *tipSetCache
|
||||
lk sync.Mutex
|
||||
|
||||
ready chan struct{}
|
||||
readyOnce sync.Once
|
||||
|
||||
heightEvents
|
||||
*hcEvents
|
||||
|
||||
observers []TipSetObserver
|
||||
}
|
||||
|
||||
func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) (*Events, error) {
|
||||
cache := newCache(api, gcConfidence)
|
||||
func NewEventsWithConfidence(ctx context.Context, api EventAPI, gcConfidence abi.ChainEpoch) *Events {
|
||||
tsc := newTSCache(gcConfidence, api)
|
||||
|
||||
ob := newObserver(cache, gcConfidence)
|
||||
if err := ob.start(ctx); err != nil {
|
||||
return nil, err
|
||||
e := &Events{
|
||||
api: api,
|
||||
|
||||
tsc: tsc,
|
||||
|
||||
heightEvents: heightEvents{
|
||||
tsc: tsc,
|
||||
ctx: ctx,
|
||||
gcConfidence: gcConfidence,
|
||||
|
||||
heightTriggers: map[uint64]*heightHandler{},
|
||||
htTriggerHeights: map[abi.ChainEpoch][]uint64{},
|
||||
htHeights: map[abi.ChainEpoch][]uint64{},
|
||||
},
|
||||
|
||||
hcEvents: newHCEvents(ctx, api, tsc, uint64(gcConfidence)),
|
||||
ready: make(chan struct{}),
|
||||
observers: []TipSetObserver{},
|
||||
}
|
||||
|
||||
he := newHeightEvents(cache, ob, gcConfidence)
|
||||
headChange := newHCEvents(cache, ob)
|
||||
go e.listenHeadChanges(ctx)
|
||||
|
||||
return &Events{ob, he, headChange}, nil
|
||||
// Wait for the first tipset to be seen or bail if shutting down
|
||||
select {
|
||||
case <-e.ready:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
func NewEvents(ctx context.Context, api EventAPI) (*Events, error) {
|
||||
func NewEvents(ctx context.Context, api EventAPI) *Events {
|
||||
gcConfidence := 2 * build.ForkLengthThreshold
|
||||
return NewEventsWithConfidence(ctx, api, gcConfidence)
|
||||
}
|
||||
|
||||
func (e *Events) listenHeadChanges(ctx context.Context) {
|
||||
for {
|
||||
if err := e.listenHeadChangesOnce(ctx); err != nil {
|
||||
log.Errorf("listen head changes errored: %s", err)
|
||||
} else {
|
||||
log.Warn("listenHeadChanges quit")
|
||||
}
|
||||
select {
|
||||
case <-build.Clock.After(time.Second):
|
||||
case <-ctx.Done():
|
||||
log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err())
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("restarting listenHeadChanges")
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Events) listenHeadChangesOnce(ctx context.Context) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
notifs, err := e.api.ChainNotify(ctx)
|
||||
if err != nil {
|
||||
// Retry is handled by caller
|
||||
return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err)
|
||||
}
|
||||
|
||||
var cur []*api.HeadChange
|
||||
var ok bool
|
||||
|
||||
// Wait for first tipset or bail
|
||||
select {
|
||||
case cur, ok = <-notifs:
|
||||
if !ok {
|
||||
return xerrors.Errorf("notification channel closed")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if len(cur) != 1 {
|
||||
return xerrors.Errorf("unexpected initial head notification length: %d", len(cur))
|
||||
}
|
||||
|
||||
if cur[0].Type != store.HCCurrent {
|
||||
return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type)
|
||||
}
|
||||
|
||||
if err := e.tsc.add(cur[0].Val); err != nil {
|
||||
log.Warnf("tsc.add: adding current tipset failed: %v", err)
|
||||
}
|
||||
|
||||
e.readyOnce.Do(func() {
|
||||
e.lastTs = cur[0].Val
|
||||
// Signal that we have seen first tipset
|
||||
close(e.ready)
|
||||
})
|
||||
|
||||
for notif := range notifs {
|
||||
var rev, app []*types.TipSet
|
||||
for _, notif := range notif {
|
||||
switch notif.Type {
|
||||
case store.HCRevert:
|
||||
rev = append(rev, notif.Val)
|
||||
case store.HCApply:
|
||||
app = append(app, notif.Val)
|
||||
default:
|
||||
log.Warnf("unexpected head change notification type: '%s'", notif.Type)
|
||||
}
|
||||
}
|
||||
|
||||
if err := e.headChange(ctx, rev, app); err != nil {
|
||||
log.Warnf("headChange failed: %s", err)
|
||||
}
|
||||
|
||||
// sync with fake chainstore (for tests)
|
||||
if fcs, ok := e.api.(interface{ notifDone() }); ok {
|
||||
fcs.notifDone()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Events) headChange(ctx context.Context, rev, app []*types.TipSet) error {
|
||||
if len(app) == 0 {
|
||||
return xerrors.New("events.headChange expected at least one applied tipset")
|
||||
}
|
||||
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
|
||||
if err := e.headChangeAt(rev, app); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := e.observeChanges(ctx, rev, app); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.processHeadChangeEvent(rev, app)
|
||||
}
|
||||
|
||||
// A TipSetObserver receives notifications of tipsets
|
||||
type TipSetObserver interface {
|
||||
Apply(ctx context.Context, ts *types.TipSet) error
|
||||
Revert(ctx context.Context, ts *types.TipSet) error
|
||||
}
|
||||
|
||||
// TODO: add a confidence level so we can have observers with difference levels of confidence
|
||||
func (e *Events) Observe(obs TipSetObserver) error {
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
e.observers = append(e.observers, obs)
|
||||
return nil
|
||||
}
|
||||
|
||||
// observeChanges expects caller to hold e.lk
|
||||
func (e *Events) observeChanges(ctx context.Context, rev, app []*types.TipSet) error {
|
||||
for _, ts := range rev {
|
||||
for _, o := range e.observers {
|
||||
_ = o.Revert(ctx, ts)
|
||||
}
|
||||
}
|
||||
|
||||
for _, ts := range app {
|
||||
for _, o := range e.observers {
|
||||
_ = o.Apply(ctx, ts)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+152
-102
@@ -32,7 +32,7 @@ type eventData interface{}
|
||||
// `prevTs` is the previous tipset, eg the "from" tipset for a state change.
|
||||
// `ts` is the event tipset, eg the tipset in which the `msg` is included.
|
||||
// `curH`-`ts.Height` = `confidence`
|
||||
type EventHandler func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
|
||||
type EventHandler func(data eventData, prevTs, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
|
||||
|
||||
// CheckFunc is used for atomicity guarantees. If the condition the callbacks
|
||||
// wait for has already happened in tipset `ts`
|
||||
@@ -40,7 +40,7 @@ type EventHandler func(ctx context.Context, data eventData, prevTs, ts *types.Ti
|
||||
// If `done` is true, timeout won't be triggered
|
||||
// If `more` is false, no messages will be sent to EventHandler (RevertHandler
|
||||
// may still be called)
|
||||
type CheckFunc func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error)
|
||||
type CheckFunc func(ts *types.TipSet) (done bool, more bool, err error)
|
||||
|
||||
// Keep track of information for an event handler
|
||||
type handlerInfo struct {
|
||||
@@ -57,9 +57,10 @@ type handlerInfo struct {
|
||||
// until the required confidence is reached
|
||||
type queuedEvent struct {
|
||||
trigger triggerID
|
||||
data eventData
|
||||
|
||||
prevTipset, tipset *types.TipSet
|
||||
prevH abi.ChainEpoch
|
||||
h abi.ChainEpoch
|
||||
data eventData
|
||||
|
||||
called bool
|
||||
}
|
||||
@@ -67,17 +68,19 @@ type queuedEvent struct {
|
||||
// Manages chain head change events, which may be forward (new tipset added to
|
||||
// chain) or backward (chain branch discarded in favour of heavier branch)
|
||||
type hcEvents struct {
|
||||
cs EventAPI
|
||||
cs EventAPI
|
||||
tsc *tipSetCache
|
||||
ctx context.Context
|
||||
gcConfidence uint64
|
||||
|
||||
lk sync.Mutex
|
||||
lastTs *types.TipSet
|
||||
|
||||
lk sync.Mutex
|
||||
|
||||
ctr triggerID
|
||||
|
||||
// TODO: get rid of trigger IDs and just use pointers as keys.
|
||||
triggers map[triggerID]*handlerInfo
|
||||
|
||||
// TODO: instead of scheduling events in the future, look at the chain in the past. We can sip the "confidence" queue entirely.
|
||||
// maps block heights to events
|
||||
// [triggerH][msgH][event]
|
||||
confQueue map[triggerH]map[msgH][]*queuedEvent
|
||||
@@ -92,77 +95,83 @@ type hcEvents struct {
|
||||
watcherEvents
|
||||
}
|
||||
|
||||
func newHCEvents(api EventAPI, obs *observer) *hcEvents {
|
||||
e := &hcEvents{
|
||||
cs: api,
|
||||
func newHCEvents(ctx context.Context, cs EventAPI, tsc *tipSetCache, gcConfidence uint64) *hcEvents {
|
||||
e := hcEvents{
|
||||
ctx: ctx,
|
||||
cs: cs,
|
||||
tsc: tsc,
|
||||
gcConfidence: gcConfidence,
|
||||
|
||||
confQueue: map[triggerH]map[msgH][]*queuedEvent{},
|
||||
revertQueue: map[msgH][]triggerH{},
|
||||
triggers: map[triggerID]*handlerInfo{},
|
||||
timeouts: map[abi.ChainEpoch]map[triggerID]int{},
|
||||
}
|
||||
|
||||
e.messageEvents = newMessageEvents(e, api)
|
||||
e.watcherEvents = newWatcherEvents(e, api)
|
||||
e.messageEvents = newMessageEvents(ctx, &e, cs)
|
||||
e.watcherEvents = newWatcherEvents(ctx, &e, cs)
|
||||
|
||||
// We need to take the lock as the observer could immediately try calling us.
|
||||
e.lk.Lock()
|
||||
e.lastTs = obs.Observe((*hcEventsObserver)(e))
|
||||
e.lk.Unlock()
|
||||
|
||||
return e
|
||||
return &e
|
||||
}
|
||||
|
||||
type hcEventsObserver hcEvents
|
||||
|
||||
func (e *hcEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error {
|
||||
// Called when there is a change to the head with tipsets to be
|
||||
// reverted / applied
|
||||
func (e *hcEvents) processHeadChangeEvent(rev, app []*types.TipSet) error {
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
|
||||
defer func() { e.lastTs = to }()
|
||||
|
||||
// Check if the head change caused any state changes that we were
|
||||
// waiting for
|
||||
stateChanges := e.checkStateChanges(from, to)
|
||||
|
||||
// Queue up calls until there have been enough blocks to reach
|
||||
// confidence on the state changes
|
||||
for tid, data := range stateChanges {
|
||||
e.queueForConfidence(tid, data, from, to)
|
||||
for _, ts := range rev {
|
||||
e.handleReverts(ts)
|
||||
e.lastTs = ts
|
||||
}
|
||||
|
||||
// Check if the head change included any new message calls
|
||||
newCalls := e.checkNewCalls(ctx, from, to)
|
||||
for _, ts := range app {
|
||||
// Check if the head change caused any state changes that we were
|
||||
// waiting for
|
||||
stateChanges := e.watcherEvents.checkStateChanges(e.lastTs, ts)
|
||||
|
||||
// Queue up calls until there have been enough blocks to reach
|
||||
// confidence on the message calls
|
||||
for tid, calls := range newCalls {
|
||||
for _, data := range calls {
|
||||
e.queueForConfidence(tid, data, nil, to)
|
||||
// Queue up calls until there have been enough blocks to reach
|
||||
// confidence on the state changes
|
||||
for tid, data := range stateChanges {
|
||||
e.queueForConfidence(tid, data, e.lastTs, ts)
|
||||
}
|
||||
|
||||
// Check if the head change included any new message calls
|
||||
newCalls, err := e.messageEvents.checkNewCalls(ts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Queue up calls until there have been enough blocks to reach
|
||||
// confidence on the message calls
|
||||
for tid, calls := range newCalls {
|
||||
for _, data := range calls {
|
||||
e.queueForConfidence(tid, data, nil, ts)
|
||||
}
|
||||
}
|
||||
|
||||
for at := e.lastTs.Height(); at <= ts.Height(); at++ {
|
||||
// Apply any queued events and timeouts that were targeted at the
|
||||
// current chain height
|
||||
e.applyWithConfidence(ts, at)
|
||||
e.applyTimeouts(ts)
|
||||
}
|
||||
|
||||
// Update the latest known tipset
|
||||
e.lastTs = ts
|
||||
}
|
||||
|
||||
for at := from.Height() + 1; at <= to.Height(); at++ {
|
||||
// Apply any queued events and timeouts that were targeted at the
|
||||
// current chain height
|
||||
e.applyWithConfidence(ctx, at)
|
||||
e.applyTimeouts(ctx, at, to)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *hcEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error {
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
|
||||
defer func() { e.lastTs = to }()
|
||||
|
||||
reverts, ok := e.revertQueue[from.Height()]
|
||||
func (e *hcEvents) handleReverts(ts *types.TipSet) {
|
||||
reverts, ok := e.revertQueue[ts.Height()]
|
||||
if !ok {
|
||||
return nil // nothing to do
|
||||
return // nothing to do
|
||||
}
|
||||
|
||||
for _, triggerH := range reverts {
|
||||
toRevert := e.confQueue[triggerH][from.Height()]
|
||||
toRevert := e.confQueue[triggerH][ts.Height()]
|
||||
for _, event := range toRevert {
|
||||
if !event.called {
|
||||
continue // event wasn't apply()-ied yet
|
||||
@@ -170,21 +179,24 @@ func (e *hcEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) e
|
||||
|
||||
trigger := e.triggers[event.trigger]
|
||||
|
||||
if err := trigger.revert(ctx, from); err != nil {
|
||||
log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", from.Height(), triggerH, err)
|
||||
if err := trigger.revert(e.ctx, ts); err != nil {
|
||||
log.Errorf("reverting chain trigger (@H %d, triggered @ %d) failed: %s", ts.Height(), triggerH, err)
|
||||
}
|
||||
}
|
||||
delete(e.confQueue[triggerH], from.Height())
|
||||
delete(e.confQueue[triggerH], ts.Height())
|
||||
}
|
||||
delete(e.revertQueue, from.Height())
|
||||
return nil
|
||||
delete(e.revertQueue, ts.Height())
|
||||
}
|
||||
|
||||
// Queue up events until the chain has reached a height that reflects the
|
||||
// desired confidence
|
||||
func (e *hcEventsObserver) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) {
|
||||
func (e *hcEvents) queueForConfidence(trigID uint64, data eventData, prevTs, ts *types.TipSet) {
|
||||
trigger := e.triggers[trigID]
|
||||
|
||||
prevH := NoHeight
|
||||
if prevTs != nil {
|
||||
prevH = prevTs.Height()
|
||||
}
|
||||
appliedH := ts.Height()
|
||||
|
||||
triggerH := appliedH + abi.ChainEpoch(trigger.confidence)
|
||||
@@ -196,23 +208,28 @@ func (e *hcEventsObserver) queueForConfidence(trigID uint64, data eventData, pre
|
||||
}
|
||||
|
||||
byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{
|
||||
trigger: trigID,
|
||||
data: data,
|
||||
tipset: ts,
|
||||
prevTipset: prevTs,
|
||||
trigger: trigID,
|
||||
prevH: prevH,
|
||||
h: appliedH,
|
||||
data: data,
|
||||
})
|
||||
|
||||
e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH)
|
||||
}
|
||||
|
||||
// Apply any events that were waiting for this chain height for confidence
|
||||
func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.ChainEpoch) {
|
||||
func (e *hcEvents) applyWithConfidence(ts *types.TipSet, height abi.ChainEpoch) {
|
||||
byOrigH, ok := e.confQueue[height]
|
||||
if !ok {
|
||||
return // no triggers at this height
|
||||
}
|
||||
|
||||
for origH, events := range byOrigH {
|
||||
triggerTs, err := e.tsc.get(origH)
|
||||
if err != nil {
|
||||
log.Errorf("events: applyWithConfidence didn't find tipset for event; wanted %d; current %d", origH, height)
|
||||
}
|
||||
|
||||
for _, event := range events {
|
||||
if event.called {
|
||||
continue
|
||||
@@ -223,7 +240,18 @@ func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.C
|
||||
continue
|
||||
}
|
||||
|
||||
more, err := trigger.handle(ctx, event.data, event.prevTipset, event.tipset, height)
|
||||
// Previous tipset - this is relevant for example in a state change
|
||||
// from one tipset to another
|
||||
var prevTs *types.TipSet
|
||||
if event.prevH != NoHeight {
|
||||
prevTs, err = e.tsc.get(event.prevH)
|
||||
if err != nil {
|
||||
log.Errorf("events: applyWithConfidence didn't find tipset for previous event; wanted %d; current %d", event.prevH, height)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
more, err := trigger.handle(event.data, prevTs, triggerTs, height)
|
||||
if err != nil {
|
||||
log.Errorf("chain trigger (@H %d, triggered @ %d) failed: %s", origH, height, err)
|
||||
continue // don't revert failed calls
|
||||
@@ -242,8 +270,8 @@ func (e *hcEventsObserver) applyWithConfidence(ctx context.Context, height abi.C
|
||||
}
|
||||
|
||||
// Apply any timeouts that expire at this height
|
||||
func (e *hcEventsObserver) applyTimeouts(ctx context.Context, at abi.ChainEpoch, ts *types.TipSet) {
|
||||
triggers, ok := e.timeouts[at]
|
||||
func (e *hcEvents) applyTimeouts(ts *types.TipSet) {
|
||||
triggers, ok := e.timeouts[ts.Height()]
|
||||
if !ok {
|
||||
return // nothing to do
|
||||
}
|
||||
@@ -257,15 +285,14 @@ func (e *hcEventsObserver) applyTimeouts(ctx context.Context, at abi.ChainEpoch,
|
||||
continue
|
||||
}
|
||||
|
||||
// This should be cached.
|
||||
timeoutTs, err := e.cs.ChainGetTipSetAfterHeight(ctx, at-abi.ChainEpoch(trigger.confidence), ts.Key())
|
||||
timeoutTs, err := e.tsc.get(ts.Height() - abi.ChainEpoch(trigger.confidence))
|
||||
if err != nil {
|
||||
log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", at-abi.ChainEpoch(trigger.confidence), at)
|
||||
log.Errorf("events: applyTimeouts didn't find tipset for event; wanted %d; current %d", ts.Height()-abi.ChainEpoch(trigger.confidence), ts.Height())
|
||||
}
|
||||
|
||||
more, err := trigger.handle(ctx, nil, nil, timeoutTs, at)
|
||||
more, err := trigger.handle(nil, nil, timeoutTs, ts.Height())
|
||||
if err != nil {
|
||||
log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), at, err)
|
||||
log.Errorf("chain trigger (call @H %d, called @ %d) failed: %s", timeoutTs.Height(), ts.Height(), err)
|
||||
continue // don't revert failed calls
|
||||
}
|
||||
|
||||
@@ -279,19 +306,24 @@ func (e *hcEventsObserver) applyTimeouts(ctx context.Context, at abi.ChainEpoch,
|
||||
// - RevertHandler: called if the chain head changes causing the event to revert
|
||||
// - confidence: wait this many tipsets before calling EventHandler
|
||||
// - timeout: at this chain height, timeout on waiting for this event
|
||||
func (e *hcEvents) onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) {
|
||||
func (e *hcEvents) onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error) {
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
|
||||
// Check if the event has already occurred
|
||||
done, more, err := check(ctx, e.lastTs)
|
||||
ts, err := e.tsc.best()
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("called check error (h: %d): %w", e.lastTs.Height(), err)
|
||||
return 0, xerrors.Errorf("error getting best tipset: %w", err)
|
||||
}
|
||||
done, more, err := check(ts)
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("called check error (h: %d): %w", ts.Height(), err)
|
||||
}
|
||||
if done {
|
||||
timeout = NoTimeout
|
||||
}
|
||||
|
||||
// Create a trigger for the event
|
||||
id := e.ctr
|
||||
e.ctr++
|
||||
|
||||
@@ -319,11 +351,12 @@ func (e *hcEvents) onHeadChanged(ctx context.Context, check CheckFunc, hnd Event
|
||||
// headChangeAPI is used to allow the composed event APIs to call back to hcEvents
|
||||
// to listen for changes
|
||||
type headChangeAPI interface {
|
||||
onHeadChanged(ctx context.Context, check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error)
|
||||
onHeadChanged(check CheckFunc, hnd EventHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch) (triggerID, error)
|
||||
}
|
||||
|
||||
// watcherEvents watches for a state change
|
||||
type watcherEvents struct {
|
||||
ctx context.Context
|
||||
cs EventAPI
|
||||
hcAPI headChangeAPI
|
||||
|
||||
@@ -331,8 +364,9 @@ type watcherEvents struct {
|
||||
matchers map[triggerID]StateMatchFunc
|
||||
}
|
||||
|
||||
func newWatcherEvents(hcAPI headChangeAPI, cs EventAPI) watcherEvents {
|
||||
func newWatcherEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) watcherEvents {
|
||||
return watcherEvents{
|
||||
ctx: ctx,
|
||||
cs: cs,
|
||||
hcAPI: hcAPI,
|
||||
matchers: make(map[triggerID]StateMatchFunc),
|
||||
@@ -388,7 +422,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error)
|
||||
// * `StateChangeHandler` is called when the specified state change was observed
|
||||
// on-chain, and a confidence threshold was reached, or the specified `timeout`
|
||||
// height was reached with no state change observed. When this callback is
|
||||
// invoked on a timeout, `oldTs` and `states are set to nil.
|
||||
// invoked on a timeout, `oldState` and `newState` are set to nil.
|
||||
// This callback returns a boolean specifying whether further notifications
|
||||
// should be sent, like `more` return param from `CheckFunc` above.
|
||||
//
|
||||
@@ -401,7 +435,7 @@ type StateMatchFunc func(oldTs, newTs *types.TipSet) (bool, StateChange, error)
|
||||
// the state change is queued up until the confidence interval has elapsed (and
|
||||
// `StateChangeHandler` is called)
|
||||
func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf StateMatchFunc) error {
|
||||
hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
|
||||
hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
|
||||
states, ok := data.(StateChange)
|
||||
if data != nil && !ok {
|
||||
panic("expected StateChange")
|
||||
@@ -410,7 +444,7 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler,
|
||||
return scHnd(prevTs, ts, states, height)
|
||||
}
|
||||
|
||||
id, err := we.hcAPI.onHeadChanged(context.TODO(), check, hnd, rev, confidence, timeout)
|
||||
id, err := we.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -424,6 +458,7 @@ func (we *watcherEvents) StateChanged(check CheckFunc, scHnd StateChangeHandler,
|
||||
|
||||
// messageEvents watches for message calls to actors
|
||||
type messageEvents struct {
|
||||
ctx context.Context
|
||||
cs EventAPI
|
||||
hcAPI headChangeAPI
|
||||
|
||||
@@ -431,8 +466,9 @@ type messageEvents struct {
|
||||
matchers map[triggerID]MsgMatchFunc
|
||||
}
|
||||
|
||||
func newMessageEvents(hcAPI headChangeAPI, cs EventAPI) messageEvents {
|
||||
func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) messageEvents {
|
||||
return messageEvents{
|
||||
ctx: ctx,
|
||||
cs: cs,
|
||||
hcAPI: hcAPI,
|
||||
matchers: make(map[triggerID]MsgMatchFunc),
|
||||
@@ -440,13 +476,19 @@ func newMessageEvents(hcAPI headChangeAPI, cs EventAPI) messageEvents {
|
||||
}
|
||||
|
||||
// Check if there are any new actor calls
|
||||
func (me *messageEvents) checkNewCalls(ctx context.Context, from, to *types.TipSet) map[triggerID][]eventData {
|
||||
func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID][]eventData, error) {
|
||||
pts, err := me.cs.ChainGetTipSet(me.ctx, ts.Parents()) // we actually care about messages in the parent tipset here
|
||||
if err != nil {
|
||||
log.Errorf("getting parent tipset in checkNewCalls: %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
me.lk.RLock()
|
||||
defer me.lk.RUnlock()
|
||||
|
||||
// For each message in the tipset
|
||||
res := make(map[triggerID][]eventData)
|
||||
me.messagesForTs(from, func(msg *types.Message) {
|
||||
me.messagesForTs(pts, func(msg *types.Message) {
|
||||
// TODO: provide receipts
|
||||
|
||||
// Run each trigger's matcher against the message
|
||||
@@ -465,32 +507,40 @@ func (me *messageEvents) checkNewCalls(ctx context.Context, from, to *types.TipS
|
||||
}
|
||||
})
|
||||
|
||||
return res
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Get the messages in a tipset
|
||||
func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Message)) {
|
||||
seen := map[cid.Cid]struct{}{}
|
||||
|
||||
for i, tsb := range ts.Cids() {
|
||||
msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb)
|
||||
for _, tsb := range ts.Blocks() {
|
||||
|
||||
msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid())
|
||||
if err != nil {
|
||||
log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s",
|
||||
ts.Height(), tsb, ts.Blocks()[i].Messages, err)
|
||||
log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err)
|
||||
// this is quite bad, but probably better than missing all the other updates
|
||||
continue
|
||||
}
|
||||
for i, c := range msgs.Cids {
|
||||
// We iterate over the CIDs to avoid having to recompute them.
|
||||
_, ok := seen[c]
|
||||
|
||||
for _, m := range msgs.BlsMessages {
|
||||
_, ok := seen[m.Cid()]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
if i < len(msgs.BlsMessages) {
|
||||
consume(msgs.BlsMessages[i])
|
||||
} else {
|
||||
consume(&msgs.SecpkMessages[i-len(msgs.BlsMessages)].Message)
|
||||
seen[m.Cid()] = struct{}{}
|
||||
|
||||
consume(m)
|
||||
}
|
||||
|
||||
for _, m := range msgs.SecpkMessages {
|
||||
_, ok := seen[m.Message.Cid()]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
seen[m.Message.Cid()] = struct{}{}
|
||||
|
||||
consume(&m.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -530,14 +580,14 @@ type MsgMatchFunc func(msg *types.Message) (matched bool, err error)
|
||||
// * `MsgMatchFunc` is called against each message. If there is a match, the
|
||||
// message is queued up until the confidence interval has elapsed (and
|
||||
// `MsgHandler` is called)
|
||||
func (me *messageEvents) Called(ctx context.Context, check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error {
|
||||
hnd := func(ctx context.Context, data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
|
||||
func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, mf MsgMatchFunc) error {
|
||||
hnd := func(data eventData, prevTs, ts *types.TipSet, height abi.ChainEpoch) (bool, error) {
|
||||
msg, ok := data.(*types.Message)
|
||||
if data != nil && !ok {
|
||||
panic("expected msg")
|
||||
}
|
||||
|
||||
ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true)
|
||||
ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -549,7 +599,7 @@ func (me *messageEvents) Called(ctx context.Context, check CheckFunc, msgHnd Msg
|
||||
return msgHnd(msg, &ml.Receipt, ts, height)
|
||||
}
|
||||
|
||||
id, err := me.hcAPI.onHeadChanged(ctx, check, hnd, rev, confidence, timeout)
|
||||
id, err := me.hcAPI.onHeadChanged(check, hnd, rev, confidence, timeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -563,5 +613,5 @@ func (me *messageEvents) Called(ctx context.Context, check CheckFunc, msgHnd Msg
|
||||
|
||||
// Convenience function for checking and matching messages
|
||||
func (me *messageEvents) CalledMsg(ctx context.Context, hnd MsgHandler, rev RevertHandler, confidence int, timeout abi.ChainEpoch, msg types.ChainMsg) error {
|
||||
return me.Called(ctx, me.CheckMsg(msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage()))
|
||||
return me.Called(me.CheckMsg(ctx, msg, hnd), hnd, rev, confidence, timeout, me.MatchMsg(msg.VMMessage()))
|
||||
}
|
||||
|
||||
+176
-212
@@ -11,235 +11,199 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
type heightHandler struct {
|
||||
ts *types.TipSet
|
||||
height abi.ChainEpoch
|
||||
called bool
|
||||
|
||||
handle HeightHandler
|
||||
revert RevertHandler
|
||||
}
|
||||
|
||||
type heightEvents struct {
|
||||
api EventAPI
|
||||
lk sync.Mutex
|
||||
tsc *tipSetCache
|
||||
gcConfidence abi.ChainEpoch
|
||||
|
||||
lk sync.Mutex
|
||||
head *types.TipSet
|
||||
tsHeights, triggerHeights map[abi.ChainEpoch][]*heightHandler
|
||||
lastGc abi.ChainEpoch //nolint:structcheck
|
||||
ctr triggerID
|
||||
|
||||
heightTriggers map[triggerID]*heightHandler
|
||||
|
||||
htTriggerHeights map[triggerH][]triggerID
|
||||
htHeights map[msgH][]triggerID
|
||||
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func newHeightEvents(api EventAPI, obs *observer, gcConfidence abi.ChainEpoch) *heightEvents {
|
||||
he := &heightEvents{
|
||||
api: api,
|
||||
gcConfidence: gcConfidence,
|
||||
tsHeights: map[abi.ChainEpoch][]*heightHandler{},
|
||||
triggerHeights: map[abi.ChainEpoch][]*heightHandler{},
|
||||
func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
|
||||
ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange")
|
||||
defer span.End()
|
||||
span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height())))
|
||||
span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev))))
|
||||
span.AddAttributes(trace.Int64Attribute("applies", int64(len(app))))
|
||||
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
for _, ts := range rev {
|
||||
// TODO: log error if h below gcconfidence
|
||||
// revert height-based triggers
|
||||
|
||||
revert := func(h abi.ChainEpoch, ts *types.TipSet) {
|
||||
for _, tid := range e.htHeights[h] {
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
|
||||
|
||||
rev := e.heightTriggers[tid].revert
|
||||
e.lk.Unlock()
|
||||
err := rev(ctx, ts)
|
||||
e.lk.Lock()
|
||||
e.heightTriggers[tid].called = false
|
||||
|
||||
span.End()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("reverting chain trigger (@H %d): %s", h, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
revert(ts.Height(), ts)
|
||||
|
||||
subh := ts.Height() - 1
|
||||
for {
|
||||
cts, err := e.tsc.get(subh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cts != nil {
|
||||
break
|
||||
}
|
||||
|
||||
revert(subh, ts)
|
||||
subh--
|
||||
}
|
||||
|
||||
if err := e.tsc.revert(ts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
he.lk.Lock()
|
||||
he.head = obs.Observe((*heightEventsObserver)(he))
|
||||
he.lk.Unlock()
|
||||
return he
|
||||
|
||||
for i := range app {
|
||||
ts := app[i]
|
||||
|
||||
if err := e.tsc.add(ts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// height triggers
|
||||
|
||||
apply := func(h abi.ChainEpoch, ts *types.TipSet) error {
|
||||
for _, tid := range e.htTriggerHeights[h] {
|
||||
hnd := e.heightTriggers[tid]
|
||||
if hnd.called {
|
||||
return nil
|
||||
}
|
||||
|
||||
triggerH := h - abi.ChainEpoch(hnd.confidence)
|
||||
|
||||
incTs, err := e.tsc.getNonNull(triggerH)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightApply")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", false))
|
||||
handle := hnd.handle
|
||||
e.lk.Unlock()
|
||||
err = handle(ctx, incTs, h)
|
||||
e.lk.Lock()
|
||||
hnd.called = true
|
||||
span.End()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", triggerH, ts.Height(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := apply(ts.Height(), ts); err != nil {
|
||||
return err
|
||||
}
|
||||
subh := ts.Height() - 1
|
||||
for {
|
||||
cts, err := e.tsc.get(subh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cts != nil {
|
||||
break
|
||||
}
|
||||
|
||||
if err := apply(subh, ts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
subh--
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChainAt invokes the specified `HeightHandler` when the chain reaches the
|
||||
// specified height+confidence threshold. If the chain is rolled-back under the
|
||||
// specified height, `RevertHandler` will be called.
|
||||
//
|
||||
// ts passed to handlers is the tipset at the specified epoch, or above if lower tipsets were null.
|
||||
//
|
||||
// The context governs cancellations of this call, it won't cancel the event handler.
|
||||
func (e *heightEvents) ChainAt(ctx context.Context, hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error {
|
||||
if abi.ChainEpoch(confidence) > e.gcConfidence {
|
||||
// Need this to be able to GC effectively.
|
||||
return xerrors.Errorf("confidence cannot be greater than gcConfidence: %d > %d", confidence, e.gcConfidence)
|
||||
// ts passed to handlers is the tipset at the specified, or above, if lower tipsets were null
|
||||
func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error {
|
||||
e.lk.Lock() // Tricky locking, check your locks if you modify this function!
|
||||
|
||||
best, err := e.tsc.best()
|
||||
if err != nil {
|
||||
e.lk.Unlock()
|
||||
return xerrors.Errorf("error getting best tipset: %w", err)
|
||||
}
|
||||
handler := &heightHandler{
|
||||
height: h,
|
||||
|
||||
bestH := best.Height()
|
||||
if bestH >= h+abi.ChainEpoch(confidence) {
|
||||
ts, err := e.tsc.getNonNull(h)
|
||||
if err != nil {
|
||||
log.Warnf("events.ChainAt: calling HandleFunc with nil tipset, not found in cache: %s", err)
|
||||
}
|
||||
|
||||
e.lk.Unlock()
|
||||
ctx, span := trace.StartSpan(e.ctx, "events.HeightApply")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", true))
|
||||
|
||||
err = hnd(ctx, ts, bestH)
|
||||
span.End()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.lk.Lock()
|
||||
best, err = e.tsc.best()
|
||||
if err != nil {
|
||||
e.lk.Unlock()
|
||||
return xerrors.Errorf("error getting best tipset: %w", err)
|
||||
}
|
||||
bestH = best.Height()
|
||||
}
|
||||
|
||||
defer e.lk.Unlock()
|
||||
|
||||
if bestH >= h+abi.ChainEpoch(confidence)+e.gcConfidence {
|
||||
return nil
|
||||
}
|
||||
|
||||
triggerAt := h + abi.ChainEpoch(confidence)
|
||||
|
||||
id := e.ctr
|
||||
e.ctr++
|
||||
|
||||
e.heightTriggers[id] = &heightHandler{
|
||||
confidence: confidence,
|
||||
|
||||
handle: hnd,
|
||||
revert: rev,
|
||||
}
|
||||
triggerAt := h + abi.ChainEpoch(confidence)
|
||||
|
||||
// Here we try to jump onto a moving train. To avoid stopping the train, we release the lock
|
||||
// while calling the API and/or the trigger functions. Unfortunately, it's entirely possible
|
||||
// (although unlikely) to go back and forth across the trigger heights, so we need to keep
|
||||
// going back and forth here till we're synced.
|
||||
//
|
||||
// TODO: Consider using a worker goroutine so we can just drop the handler in a channel? The
|
||||
// downside is that we'd either need a tipset cache, or we'd need to potentially fetch
|
||||
// tipsets in-line inside the event loop.
|
||||
e.lk.Lock()
|
||||
for {
|
||||
head := e.head
|
||||
if head.Height() >= h {
|
||||
// Head is past the handler height. We at least need to stash the tipset to
|
||||
// avoid doing this from the main event loop.
|
||||
e.lk.Unlock()
|
||||
e.htHeights[h] = append(e.htHeights[h], id)
|
||||
e.htTriggerHeights[triggerAt] = append(e.htTriggerHeights[triggerAt], id)
|
||||
|
||||
var ts *types.TipSet
|
||||
if head.Height() == h {
|
||||
ts = head
|
||||
} else {
|
||||
var err error
|
||||
ts, err = e.api.ChainGetTipSetAfterHeight(ctx, handler.height, head.Key())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("events.ChainAt: failed to get tipset: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// If we've applied the handler on the wrong tipset, revert.
|
||||
if handler.called && !ts.Equals(handler.ts) {
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", true))
|
||||
err := handler.revert(ctx, handler.ts)
|
||||
span.End()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.called = false
|
||||
}
|
||||
|
||||
// Save the tipset.
|
||||
handler.ts = ts
|
||||
|
||||
// If we've reached confidence and haven't called, call.
|
||||
if !handler.called && head.Height() >= triggerAt {
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightApply")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", true))
|
||||
err := handler.handle(ctx, handler.ts, head.Height())
|
||||
span.End()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
handler.called = true
|
||||
|
||||
// If we've reached gcConfidence, return without saving anything.
|
||||
if head.Height() >= h+e.gcConfidence {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
e.lk.Lock()
|
||||
} else if handler.called {
|
||||
// We're not passed the head (anymore) but have applied the handler. Revert, try again.
|
||||
e.lk.Unlock()
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", true))
|
||||
err := handler.revert(ctx, handler.ts)
|
||||
span.End()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.called = false
|
||||
e.lk.Lock()
|
||||
} // otherwise, we changed heads but the change didn't matter.
|
||||
|
||||
// If we managed to get through this without the head changing, we're finally done.
|
||||
if head.Equals(e.head) {
|
||||
e.triggerHeights[triggerAt] = append(e.triggerHeights[triggerAt], handler)
|
||||
e.tsHeights[h] = append(e.tsHeights[h], handler)
|
||||
e.lk.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Updates the head and garbage collects if we're 2x over our garbage collection confidence period.
|
||||
func (e *heightEventsObserver) updateHead(h *types.TipSet) {
|
||||
e.lk.Lock()
|
||||
defer e.lk.Unlock()
|
||||
e.head = h
|
||||
|
||||
if e.head.Height() < e.lastGc+e.gcConfidence*2 {
|
||||
return
|
||||
}
|
||||
e.lastGc = h.Height()
|
||||
|
||||
targetGcHeight := e.head.Height() - e.gcConfidence
|
||||
for h := range e.tsHeights {
|
||||
if h >= targetGcHeight {
|
||||
continue
|
||||
}
|
||||
delete(e.tsHeights, h)
|
||||
}
|
||||
for h := range e.triggerHeights {
|
||||
if h >= targetGcHeight {
|
||||
continue
|
||||
}
|
||||
delete(e.triggerHeights, h)
|
||||
}
|
||||
}
|
||||
|
||||
type heightEventsObserver heightEvents
|
||||
|
||||
func (e *heightEventsObserver) Revert(ctx context.Context, from, to *types.TipSet) error {
|
||||
// Update the head first so we don't accidental skip reverting a concurrent call to ChainAt.
|
||||
e.updateHead(to)
|
||||
|
||||
// Call revert on all hights between the two tipsets, handling empty tipsets.
|
||||
for h := from.Height(); h > to.Height(); h-- {
|
||||
e.lk.Lock()
|
||||
triggers := e.tsHeights[h]
|
||||
e.lk.Unlock()
|
||||
|
||||
// 1. Triggers are only invoked from the global event loop, we don't need to hold the lock while calling.
|
||||
// 2. We only ever append to or replace the trigger slice, so it's safe to iterate over it without the lock.
|
||||
for _, handler := range triggers {
|
||||
handler.ts = nil // invalidate
|
||||
if !handler.called {
|
||||
// We haven't triggered this yet, or there has been a concurrent call to ChainAt.
|
||||
continue
|
||||
}
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
|
||||
err := handler.revert(ctx, from)
|
||||
span.End()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("reverting chain trigger (@H %d): %s", h, err)
|
||||
}
|
||||
handler.called = false
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *heightEventsObserver) Apply(ctx context.Context, from, to *types.TipSet) error {
|
||||
// Update the head first so we don't accidental skip applying a concurrent call to ChainAt.
|
||||
e.updateHead(to)
|
||||
|
||||
for h := from.Height() + 1; h <= to.Height(); h++ {
|
||||
e.lk.Lock()
|
||||
triggers := e.triggerHeights[h]
|
||||
tipsets := e.tsHeights[h]
|
||||
e.lk.Unlock()
|
||||
|
||||
// Stash the tipset for future triggers.
|
||||
for _, handler := range tipsets {
|
||||
handler.ts = to
|
||||
}
|
||||
|
||||
// Trigger the ready triggers.
|
||||
for _, handler := range triggers {
|
||||
if handler.called {
|
||||
// We may have reverted past the trigger point, but not past the call point.
|
||||
// Or there has been a concurrent call to ChainAt.
|
||||
continue
|
||||
}
|
||||
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeightApply")
|
||||
span.AddAttributes(trace.BoolAttribute("immediate", false))
|
||||
err := handler.handle(ctx, handler.ts, h)
|
||||
span.End()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("chain trigger (@H %d, called @ %d) failed: %+v", h, to.Height(), err)
|
||||
}
|
||||
|
||||
handler.called = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+397
-464
File diff suppressed because it is too large
Load Diff
@@ -1,42 +0,0 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type messageCache struct {
|
||||
api EventAPI
|
||||
|
||||
blockMsgLk sync.Mutex
|
||||
blockMsgCache *lru.ARCCache
|
||||
}
|
||||
|
||||
func newMessageCache(api EventAPI) *messageCache {
|
||||
blsMsgCache, _ := lru.NewARC(500)
|
||||
|
||||
return &messageCache{
|
||||
api: api,
|
||||
blockMsgCache: blsMsgCache,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *messageCache) ChainGetBlockMessages(ctx context.Context, blkCid cid.Cid) (*api.BlockMessages, error) {
|
||||
c.blockMsgLk.Lock()
|
||||
defer c.blockMsgLk.Unlock()
|
||||
|
||||
msgsI, ok := c.blockMsgCache.Get(blkCid)
|
||||
var err error
|
||||
if !ok {
|
||||
msgsI, err = c.api.ChainGetBlockMessages(ctx, blkCid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.blockMsgCache.Add(blkCid, msgsI)
|
||||
}
|
||||
return msgsI.(*api.BlockMessages), nil
|
||||
}
|
||||
@@ -1,255 +0,0 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"go.opencensus.io/trace"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
type observer struct {
|
||||
api EventAPI
|
||||
|
||||
gcConfidence abi.ChainEpoch
|
||||
|
||||
ready chan struct{}
|
||||
|
||||
lk sync.Mutex
|
||||
head *types.TipSet
|
||||
maxHeight abi.ChainEpoch
|
||||
observers []TipSetObserver
|
||||
}
|
||||
|
||||
func newObserver(api *cache, gcConfidence abi.ChainEpoch) *observer {
|
||||
obs := &observer{
|
||||
api: api,
|
||||
gcConfidence: gcConfidence,
|
||||
|
||||
ready: make(chan struct{}),
|
||||
observers: []TipSetObserver{},
|
||||
}
|
||||
obs.Observe(api.observer())
|
||||
return obs
|
||||
}
|
||||
|
||||
func (o *observer) start(ctx context.Context) error {
|
||||
go o.listenHeadChanges(ctx)
|
||||
|
||||
// Wait for the first tipset to be seen or bail if shutting down
|
||||
select {
|
||||
case <-o.ready:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (o *observer) listenHeadChanges(ctx context.Context) {
|
||||
for {
|
||||
if err := o.listenHeadChangesOnce(ctx); err != nil {
|
||||
log.Errorf("listen head changes errored: %s", err)
|
||||
} else {
|
||||
log.Warn("listenHeadChanges quit")
|
||||
}
|
||||
select {
|
||||
case <-build.Clock.After(time.Second):
|
||||
case <-ctx.Done():
|
||||
log.Warnf("not restarting listenHeadChanges: context error: %s", ctx.Err())
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("restarting listenHeadChanges")
|
||||
}
|
||||
}
|
||||
|
||||
func (o *observer) listenHeadChangesOnce(ctx context.Context) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
notifs, err := o.api.ChainNotify(ctx)
|
||||
if err != nil {
|
||||
// Retry is handled by caller
|
||||
return xerrors.Errorf("listenHeadChanges ChainNotify call failed: %w", err)
|
||||
}
|
||||
|
||||
var cur []*api.HeadChange
|
||||
var ok bool
|
||||
|
||||
// Wait for first tipset or bail
|
||||
select {
|
||||
case cur, ok = <-notifs:
|
||||
if !ok {
|
||||
return xerrors.Errorf("notification channel closed")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if len(cur) != 1 {
|
||||
return xerrors.Errorf("unexpected initial head notification length: %d", len(cur))
|
||||
}
|
||||
|
||||
if cur[0].Type != store.HCCurrent {
|
||||
return xerrors.Errorf("expected first head notification type to be 'current', was '%s'", cur[0].Type)
|
||||
}
|
||||
|
||||
curHead := cur[0].Val
|
||||
|
||||
o.lk.Lock()
|
||||
if o.head == nil {
|
||||
o.head = curHead
|
||||
close(o.ready)
|
||||
}
|
||||
startHead := o.head
|
||||
o.lk.Unlock()
|
||||
|
||||
if !startHead.Equals(curHead) {
|
||||
changes, err := o.api.ChainGetPath(ctx, startHead.Key(), curHead.Key())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get path from last applied tipset to head: %w", err)
|
||||
}
|
||||
|
||||
if err := o.applyChanges(ctx, changes); err != nil {
|
||||
return xerrors.Errorf("failed catch-up head changes: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
for changes := range notifs {
|
||||
if err := o.applyChanges(ctx, changes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *observer) applyChanges(ctx context.Context, changes []*api.HeadChange) error {
|
||||
// Used to wait for a prior notification round to finish (by tests)
|
||||
if len(changes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var rev, app []*types.TipSet
|
||||
for _, changes := range changes {
|
||||
switch changes.Type {
|
||||
case store.HCRevert:
|
||||
rev = append(rev, changes.Val)
|
||||
case store.HCApply:
|
||||
app = append(app, changes.Val)
|
||||
default:
|
||||
log.Errorf("unexpected head change notification type: '%s'", changes.Type)
|
||||
}
|
||||
}
|
||||
|
||||
if err := o.headChange(ctx, rev, app); err != nil {
|
||||
return xerrors.Errorf("failed to apply head changes: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *observer) headChange(ctx context.Context, rev, app []*types.TipSet) error {
|
||||
ctx, span := trace.StartSpan(ctx, "events.HeadChange")
|
||||
span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev))))
|
||||
span.AddAttributes(trace.Int64Attribute("applies", int64(len(app))))
|
||||
|
||||
o.lk.Lock()
|
||||
head := o.head
|
||||
o.lk.Unlock()
|
||||
|
||||
defer func() {
|
||||
span.AddAttributes(trace.Int64Attribute("endHeight", int64(head.Height())))
|
||||
span.End()
|
||||
}()
|
||||
|
||||
// NOTE: bailing out here if the head isn't what we expected is fine. We'll re-start the
|
||||
// entire process and handle any strange reorgs.
|
||||
for i, from := range rev {
|
||||
if !from.Equals(head) {
|
||||
return xerrors.Errorf(
|
||||
"expected to revert %s (%d), reverting %s (%d)",
|
||||
head.Key(), head.Height(), from.Key(), from.Height(),
|
||||
)
|
||||
}
|
||||
var to *types.TipSet
|
||||
if i+1 < len(rev) {
|
||||
// If we have more reverts, the next revert is the next head.
|
||||
to = rev[i+1]
|
||||
} else {
|
||||
// At the end of the revert sequenece, we need to lookup the joint tipset
|
||||
// between the revert sequence and the apply sequence.
|
||||
var err error
|
||||
to, err = o.api.ChainGetTipSet(ctx, from.Parents())
|
||||
if err != nil {
|
||||
// Well, this sucks. We'll bail and restart.
|
||||
return xerrors.Errorf("failed to get tipset when reverting due to a SetHeead: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get the current observers and atomically set the head.
|
||||
//
|
||||
// 1. We need to get the observers every time in case some registered/deregistered.
|
||||
// 2. We need to atomically set the head so new observers don't see events twice or
|
||||
// skip them.
|
||||
o.lk.Lock()
|
||||
observers := o.observers
|
||||
o.head = to
|
||||
o.lk.Unlock()
|
||||
|
||||
for _, obs := range observers {
|
||||
if err := obs.Revert(ctx, from, to); err != nil {
|
||||
log.Errorf("observer %T failed to apply tipset %s (%d) with: %s", obs, from.Key(), from.Height(), err)
|
||||
}
|
||||
}
|
||||
|
||||
if to.Height() < o.maxHeight-o.gcConfidence {
|
||||
log.Errorf("reverted past finality, from %d to %d", o.maxHeight, to.Height())
|
||||
}
|
||||
|
||||
head = to
|
||||
}
|
||||
|
||||
for _, to := range app {
|
||||
if to.Parents() != head.Key() {
|
||||
return xerrors.Errorf(
|
||||
"cannot apply %s (%d) with parents %s on top of %s (%d)",
|
||||
to.Key(), to.Height(), to.Parents(), head.Key(), head.Height(),
|
||||
)
|
||||
}
|
||||
|
||||
o.lk.Lock()
|
||||
observers := o.observers
|
||||
o.head = to
|
||||
o.lk.Unlock()
|
||||
|
||||
for _, obs := range observers {
|
||||
if err := obs.Apply(ctx, head, to); err != nil {
|
||||
log.Errorf("observer %T failed to revert tipset %s (%d) with: %s", obs, to.Key(), to.Height(), err)
|
||||
}
|
||||
}
|
||||
if to.Height() > o.maxHeight {
|
||||
o.maxHeight = to.Height()
|
||||
}
|
||||
|
||||
head = to
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Observe registers the observer, and returns the current tipset. The observer is guaranteed to
|
||||
// observe events starting at this tipset.
|
||||
//
|
||||
// Returns nil if the observer hasn't started yet (but still registers).
|
||||
func (o *observer) Observe(obs TipSetObserver) *types.TipSet {
|
||||
o.lk.Lock()
|
||||
defer o.lk.Unlock()
|
||||
o.observers = append(o.observers, obs)
|
||||
return o.head
|
||||
}
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
|
||||
miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
|
||||
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
|
||||
tutils "github.com/filecoin-project/specs-actors/v5/support/testing"
|
||||
tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
|
||||
|
||||
+75
-133
@@ -11,9 +11,7 @@ import (
|
||||
)
|
||||
|
||||
type tsCacheAPI interface {
|
||||
ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
|
||||
ChainHead(context.Context) (*types.TipSet, error)
|
||||
}
|
||||
|
||||
@@ -22,157 +20,61 @@ type tsCacheAPI interface {
|
||||
type tipSetCache struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
byKey map[types.TipSetKey]*types.TipSet
|
||||
byHeight []*types.TipSet
|
||||
start int // chain head (end)
|
||||
len int
|
||||
cache []*types.TipSet
|
||||
start int
|
||||
len int
|
||||
|
||||
storage tsCacheAPI
|
||||
}
|
||||
|
||||
func newTSCache(storage tsCacheAPI, cap abi.ChainEpoch) *tipSetCache {
|
||||
func newTSCache(cap abi.ChainEpoch, storage tsCacheAPI) *tipSetCache {
|
||||
return &tipSetCache{
|
||||
byKey: make(map[types.TipSetKey]*types.TipSet, cap),
|
||||
byHeight: make([]*types.TipSet, cap),
|
||||
start: 0,
|
||||
len: 0,
|
||||
cache: make([]*types.TipSet, cap),
|
||||
start: 0,
|
||||
len: 0,
|
||||
|
||||
storage: storage,
|
||||
}
|
||||
}
|
||||
func (tsc *tipSetCache) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
|
||||
if ts, ok := tsc.byKey[tsk]; ok {
|
||||
return ts, nil
|
||||
}
|
||||
return tsc.storage.ChainGetTipSet(ctx, tsk)
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) ChainGetTipSetByHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
|
||||
return tsc.get(ctx, height, tsk, true)
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) ChainGetTipSetAfterHeight(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
|
||||
return tsc.get(ctx, height, tsk, false)
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) get(ctx context.Context, height abi.ChainEpoch, tsk types.TipSetKey, prev bool) (*types.TipSet, error) {
|
||||
fallback := tsc.storage.ChainGetTipSetAfterHeight
|
||||
if prev {
|
||||
fallback = tsc.storage.ChainGetTipSetByHeight
|
||||
}
|
||||
tsc.mu.RLock()
|
||||
|
||||
// Nothing in the cache?
|
||||
if tsc.len == 0 {
|
||||
tsc.mu.RUnlock()
|
||||
log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height)
|
||||
return fallback(ctx, height, tsk)
|
||||
}
|
||||
|
||||
// Resolve the head.
|
||||
head := tsc.byHeight[tsc.start]
|
||||
if !tsk.IsEmpty() {
|
||||
// Not on this chain?
|
||||
var ok bool
|
||||
head, ok = tsc.byKey[tsk]
|
||||
if !ok {
|
||||
tsc.mu.RUnlock()
|
||||
return fallback(ctx, height, tsk)
|
||||
}
|
||||
}
|
||||
|
||||
headH := head.Height()
|
||||
tailH := headH - abi.ChainEpoch(tsc.len)
|
||||
|
||||
if headH == height {
|
||||
tsc.mu.RUnlock()
|
||||
return head, nil
|
||||
} else if headH < height {
|
||||
tsc.mu.RUnlock()
|
||||
// If the user doesn't pass a tsk, we assume "head" is the last tipset we processed.
|
||||
return nil, xerrors.Errorf("requested epoch is in the future")
|
||||
} else if height < tailH {
|
||||
log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tailH)
|
||||
tsc.mu.RUnlock()
|
||||
return fallback(ctx, height, head.Key())
|
||||
}
|
||||
|
||||
direction := 1
|
||||
if prev {
|
||||
direction = -1
|
||||
}
|
||||
var ts *types.TipSet
|
||||
for i := 0; i < tsc.len && ts == nil; i += direction {
|
||||
ts = tsc.byHeight[normalModulo(tsc.start-int(headH-height)+i, len(tsc.byHeight))]
|
||||
}
|
||||
tsc.mu.RUnlock()
|
||||
return ts, nil
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) ChainHead(ctx context.Context) (*types.TipSet, error) {
|
||||
tsc.mu.RLock()
|
||||
best := tsc.byHeight[tsc.start]
|
||||
tsc.mu.RUnlock()
|
||||
if best == nil {
|
||||
return tsc.storage.ChainHead(ctx)
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) add(to *types.TipSet) error {
|
||||
func (tsc *tipSetCache) add(ts *types.TipSet) error {
|
||||
tsc.mu.Lock()
|
||||
defer tsc.mu.Unlock()
|
||||
|
||||
if tsc.len > 0 {
|
||||
best := tsc.byHeight[tsc.start]
|
||||
if best.Height() >= to.Height() {
|
||||
return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.byHeight[tsc.start].Height()+1, to.Height())
|
||||
}
|
||||
if best.Key() != to.Parents() {
|
||||
return xerrors.Errorf(
|
||||
"tipSetCache.add: expected new tipset %s (%d) to follow %s (%d), its parents are %s",
|
||||
to.Key(), to.Height(), best.Key(), best.Height(), best.Parents(),
|
||||
)
|
||||
if tsc.cache[tsc.start].Height() >= ts.Height() {
|
||||
return xerrors.Errorf("tipSetCache.add: expected new tipset height to be at least %d, was %d", tsc.cache[tsc.start].Height()+1, ts.Height())
|
||||
}
|
||||
}
|
||||
|
||||
nextH := to.Height()
|
||||
nextH := ts.Height()
|
||||
if tsc.len > 0 {
|
||||
nextH = tsc.byHeight[tsc.start].Height() + 1
|
||||
nextH = tsc.cache[tsc.start].Height() + 1
|
||||
}
|
||||
|
||||
// fill null blocks
|
||||
for nextH != to.Height() {
|
||||
tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight))
|
||||
was := tsc.byHeight[tsc.start]
|
||||
if was != nil {
|
||||
tsc.byHeight[tsc.start] = nil
|
||||
delete(tsc.byKey, was.Key())
|
||||
}
|
||||
if tsc.len < len(tsc.byHeight) {
|
||||
for nextH != ts.Height() {
|
||||
tsc.start = normalModulo(tsc.start+1, len(tsc.cache))
|
||||
tsc.cache[tsc.start] = nil
|
||||
if tsc.len < len(tsc.cache) {
|
||||
tsc.len++
|
||||
}
|
||||
nextH++
|
||||
}
|
||||
|
||||
tsc.start = normalModulo(tsc.start+1, len(tsc.byHeight))
|
||||
was := tsc.byHeight[tsc.start]
|
||||
if was != nil {
|
||||
delete(tsc.byKey, was.Key())
|
||||
}
|
||||
tsc.byHeight[tsc.start] = to
|
||||
if tsc.len < len(tsc.byHeight) {
|
||||
tsc.start = normalModulo(tsc.start+1, len(tsc.cache))
|
||||
tsc.cache[tsc.start] = ts
|
||||
if tsc.len < len(tsc.cache) {
|
||||
tsc.len++
|
||||
}
|
||||
tsc.byKey[to.Key()] = to
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) revert(from *types.TipSet) error {
|
||||
func (tsc *tipSetCache) revert(ts *types.TipSet) error {
|
||||
tsc.mu.Lock()
|
||||
defer tsc.mu.Unlock()
|
||||
|
||||
return tsc.revertUnlocked(from)
|
||||
return tsc.revertUnlocked(ts)
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error {
|
||||
@@ -180,35 +82,75 @@ func (tsc *tipSetCache) revertUnlocked(ts *types.TipSet) error {
|
||||
return nil // this can happen, and it's fine
|
||||
}
|
||||
|
||||
was := tsc.byHeight[tsc.start]
|
||||
|
||||
if !was.Equals(ts) {
|
||||
if !tsc.cache[tsc.start].Equals(ts) {
|
||||
return xerrors.New("tipSetCache.revert: revert tipset didn't match cache head")
|
||||
}
|
||||
delete(tsc.byKey, was.Key())
|
||||
|
||||
tsc.byHeight[tsc.start] = nil
|
||||
tsc.start = normalModulo(tsc.start-1, len(tsc.byHeight))
|
||||
tsc.cache[tsc.start] = nil
|
||||
tsc.start = normalModulo(tsc.start-1, len(tsc.cache))
|
||||
tsc.len--
|
||||
|
||||
_ = tsc.revertUnlocked(nil) // revert null block gap
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tsc *tipSetCache) observer() TipSetObserver {
|
||||
return (*tipSetCacheObserver)(tsc)
|
||||
func (tsc *tipSetCache) getNonNull(height abi.ChainEpoch) (*types.TipSet, error) {
|
||||
for {
|
||||
ts, err := tsc.get(height)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ts != nil {
|
||||
return ts, nil
|
||||
}
|
||||
height++
|
||||
}
|
||||
}
|
||||
|
||||
type tipSetCacheObserver tipSetCache
|
||||
func (tsc *tipSetCache) get(height abi.ChainEpoch) (*types.TipSet, error) {
|
||||
tsc.mu.RLock()
|
||||
|
||||
var _ TipSetObserver = new(tipSetCacheObserver)
|
||||
if tsc.len == 0 {
|
||||
tsc.mu.RUnlock()
|
||||
log.Warnf("tipSetCache.get: cache is empty, requesting from storage (h=%d)", height)
|
||||
return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, types.EmptyTSK)
|
||||
}
|
||||
|
||||
func (tsc *tipSetCacheObserver) Apply(_ context.Context, _, to *types.TipSet) error {
|
||||
return (*tipSetCache)(tsc).add(to)
|
||||
headH := tsc.cache[tsc.start].Height()
|
||||
|
||||
if height > headH {
|
||||
tsc.mu.RUnlock()
|
||||
return nil, xerrors.Errorf("tipSetCache.get: requested tipset not in cache (req: %d, cache head: %d)", height, headH)
|
||||
}
|
||||
|
||||
clen := len(tsc.cache)
|
||||
var tail *types.TipSet
|
||||
for i := 1; i <= tsc.len; i++ {
|
||||
tail = tsc.cache[normalModulo(tsc.start-tsc.len+i, clen)]
|
||||
if tail != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if height < tail.Height() {
|
||||
tsc.mu.RUnlock()
|
||||
log.Warnf("tipSetCache.get: requested tipset not in cache, requesting from storage (h=%d; tail=%d)", height, tail.Height())
|
||||
return tsc.storage.ChainGetTipSetByHeight(context.TODO(), height, tail.Key())
|
||||
}
|
||||
|
||||
ts := tsc.cache[normalModulo(tsc.start-int(headH-height), clen)]
|
||||
tsc.mu.RUnlock()
|
||||
return ts, nil
|
||||
}
|
||||
|
||||
func (tsc *tipSetCacheObserver) Revert(ctx context.Context, from, _ *types.TipSet) error {
|
||||
return (*tipSetCache)(tsc).revert(from)
|
||||
func (tsc *tipSetCache) best() (*types.TipSet, error) {
|
||||
tsc.mu.RLock()
|
||||
best := tsc.cache[tsc.start]
|
||||
tsc.mu.RUnlock()
|
||||
if best == nil {
|
||||
return tsc.storage.ChainHead(context.TODO())
|
||||
}
|
||||
return best, nil
|
||||
}
|
||||
|
||||
func normalModulo(n, m int) int {
|
||||
|
||||
+99
-141
@@ -6,20 +6,59 @@ import (
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
type tsCacheAPIFailOnStorageCall struct {
|
||||
t *testing.T
|
||||
func TestTsCache(t *testing.T) {
|
||||
tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
|
||||
|
||||
h := abi.ChainEpoch(75)
|
||||
|
||||
a, _ := address.NewFromString("t00")
|
||||
|
||||
add := func() {
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{{
|
||||
Miner: a,
|
||||
Height: h,
|
||||
ParentStateRoot: dummyCid,
|
||||
Messages: dummyCid,
|
||||
ParentMessageReceipts: dummyCid,
|
||||
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tsc.add(ts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h++
|
||||
}
|
||||
|
||||
for i := 0; i < 9000; i++ {
|
||||
if i%90 > 60 {
|
||||
best, err := tsc.best()
|
||||
if err != nil {
|
||||
t.Fatal(err, "; i:", i)
|
||||
return
|
||||
}
|
||||
if err := tsc.revert(best); err != nil {
|
||||
t.Fatal(err, "; i:", i)
|
||||
return
|
||||
}
|
||||
h--
|
||||
} else {
|
||||
add()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
|
||||
tc.t.Fatal("storage call")
|
||||
return &types.TipSet{}, nil
|
||||
type tsCacheAPIFailOnStorageCall struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
|
||||
@@ -30,181 +69,100 @@ func (tc *tsCacheAPIFailOnStorageCall) ChainHead(ctx context.Context) (*types.Ti
|
||||
tc.t.Fatal("storage call")
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
func (tc *tsCacheAPIFailOnStorageCall) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
|
||||
tc.t.Fatal("storage call")
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
|
||||
type cacheHarness struct {
|
||||
t *testing.T
|
||||
|
||||
miner address.Address
|
||||
tsc *tipSetCache
|
||||
height abi.ChainEpoch
|
||||
}
|
||||
|
||||
func newCacheharness(t *testing.T) *cacheHarness {
|
||||
a, err := address.NewFromString("t00")
|
||||
require.NoError(t, err)
|
||||
|
||||
h := &cacheHarness{
|
||||
t: t,
|
||||
tsc: newTSCache(&tsCacheAPIFailOnStorageCall{t: t}, 50),
|
||||
height: 75,
|
||||
miner: a,
|
||||
}
|
||||
h.addWithParents(nil)
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *cacheHarness) addWithParents(parents []cid.Cid) {
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{{
|
||||
Miner: h.miner,
|
||||
Height: h.height,
|
||||
ParentStateRoot: dummyCid,
|
||||
Messages: dummyCid,
|
||||
ParentMessageReceipts: dummyCid,
|
||||
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
Parents: parents,
|
||||
}})
|
||||
require.NoError(h.t, err)
|
||||
require.NoError(h.t, h.tsc.add(ts))
|
||||
h.height++
|
||||
}
|
||||
|
||||
func (h *cacheHarness) add() {
|
||||
last, err := h.tsc.ChainHead(context.Background())
|
||||
require.NoError(h.t, err)
|
||||
h.addWithParents(last.Cids())
|
||||
}
|
||||
|
||||
func (h *cacheHarness) revert() {
|
||||
best, err := h.tsc.ChainHead(context.Background())
|
||||
require.NoError(h.t, err)
|
||||
err = h.tsc.revert(best)
|
||||
require.NoError(h.t, err)
|
||||
h.height--
|
||||
}
|
||||
|
||||
func (h *cacheHarness) skip(n abi.ChainEpoch) {
|
||||
h.height += n
|
||||
}
|
||||
|
||||
func TestTsCache(t *testing.T) {
|
||||
h := newCacheharness(t)
|
||||
|
||||
for i := 0; i < 9000; i++ {
|
||||
if i%90 > 60 {
|
||||
h.revert()
|
||||
} else {
|
||||
h.add()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsCacheNulls(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
h := newCacheharness(t)
|
||||
tsc := newTSCache(50, &tsCacheAPIFailOnStorageCall{t: t})
|
||||
|
||||
h.add()
|
||||
h.add()
|
||||
h.add()
|
||||
h.skip(5)
|
||||
h := abi.ChainEpoch(75)
|
||||
|
||||
h.add()
|
||||
h.add()
|
||||
a, _ := address.NewFromString("t00")
|
||||
add := func() {
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{{
|
||||
Miner: a,
|
||||
Height: h,
|
||||
ParentStateRoot: dummyCid,
|
||||
Messages: dummyCid,
|
||||
ParentMessageReceipts: dummyCid,
|
||||
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tsc.add(ts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h++
|
||||
}
|
||||
|
||||
best, err := h.tsc.ChainHead(ctx)
|
||||
add()
|
||||
add()
|
||||
add()
|
||||
h += 5
|
||||
|
||||
add()
|
||||
add()
|
||||
|
||||
best, err := tsc.best()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-1, best.Height())
|
||||
require.Equal(t, h-1, best.Height())
|
||||
|
||||
ts, err := h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK)
|
||||
ts, err := tsc.get(h - 1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-1, ts.Height())
|
||||
require.Equal(t, h-1, ts.Height())
|
||||
|
||||
ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-2, types.EmptyTSK)
|
||||
ts, err = tsc.get(h - 2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-2, ts.Height())
|
||||
require.Equal(t, h-2, ts.Height())
|
||||
|
||||
// Should skip the nulls and walk back to the last tipset.
|
||||
ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-3, types.EmptyTSK)
|
||||
ts, err = tsc.get(h - 3)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-8, ts.Height())
|
||||
require.Nil(t, ts)
|
||||
|
||||
ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-8, types.EmptyTSK)
|
||||
ts, err = tsc.get(h - 8)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-8, ts.Height())
|
||||
require.Equal(t, h-8, ts.Height())
|
||||
|
||||
best, err = h.tsc.ChainHead(ctx)
|
||||
best, err = tsc.best()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, h.tsc.revert(best))
|
||||
require.NoError(t, tsc.revert(best))
|
||||
|
||||
best, err = h.tsc.ChainHead(ctx)
|
||||
best, err = tsc.best()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, h.tsc.revert(best))
|
||||
require.NoError(t, tsc.revert(best))
|
||||
|
||||
best, err = h.tsc.ChainHead(ctx)
|
||||
best, err = tsc.best()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-8, best.Height())
|
||||
require.Equal(t, h-8, best.Height())
|
||||
|
||||
h.skip(50)
|
||||
h.add()
|
||||
h += 50
|
||||
add()
|
||||
|
||||
ts, err = h.tsc.ChainGetTipSetByHeight(ctx, h.height-1, types.EmptyTSK)
|
||||
ts, err = tsc.get(h - 1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, h.height-1, ts.Height())
|
||||
require.Equal(t, h-1, ts.Height())
|
||||
}
|
||||
|
||||
type tsCacheAPIStorageCallCounter struct {
|
||||
t *testing.T
|
||||
chainGetTipSetByHeight int
|
||||
chainGetTipSetAfterHeight int
|
||||
chainGetTipSet int
|
||||
chainHead int
|
||||
t *testing.T
|
||||
chainGetTipSetByHeight int
|
||||
chainHead int
|
||||
}
|
||||
|
||||
func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
|
||||
tc.chainGetTipSetByHeight++
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
|
||||
tc.chainGetTipSetAfterHeight++
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
func (tc *tsCacheAPIStorageCallCounter) ChainHead(ctx context.Context) (*types.TipSet, error) {
|
||||
tc.chainHead++
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
func (tc *tsCacheAPIStorageCallCounter) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
|
||||
tc.chainGetTipSet++
|
||||
return &types.TipSet{}, nil
|
||||
}
|
||||
|
||||
func TestTsCacheEmpty(t *testing.T) {
|
||||
// Calling best on an empty cache should just call out to the chain API
|
||||
callCounter := &tsCacheAPIStorageCallCounter{t: t}
|
||||
tsc := newTSCache(callCounter, 50)
|
||||
_, err := tsc.ChainHead(context.Background())
|
||||
tsc := newTSCache(50, callCounter)
|
||||
_, err := tsc.best()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, callCounter.chainHead)
|
||||
}
|
||||
|
||||
func TestTsCacheSkip(t *testing.T) {
|
||||
h := newCacheharness(t)
|
||||
|
||||
ts, err := types.NewTipSet([]*types.BlockHeader{{
|
||||
Miner: h.miner,
|
||||
Height: h.height,
|
||||
ParentStateRoot: dummyCid,
|
||||
Messages: dummyCid,
|
||||
ParentMessageReceipts: dummyCid,
|
||||
BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS},
|
||||
// With parents that don't match the last block.
|
||||
Parents: nil,
|
||||
}})
|
||||
require.NoError(h.t, err)
|
||||
err = h.tsc.add(ts)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
|
||||
func (me *messageEvents) CheckMsg(smsg types.ChainMsg, hnd MsgHandler) CheckFunc {
|
||||
func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd MsgHandler) CheckFunc {
|
||||
msg := smsg.VMMessage()
|
||||
|
||||
return func(ctx context.Context, ts *types.TipSet) (done bool, more bool, err error) {
|
||||
return func(ts *types.TipSet) (done bool, more bool, err error) {
|
||||
fa, err := me.cs.StateGetActor(ctx, msg.From, ts.Key())
|
||||
if err != nil {
|
||||
return false, true, err
|
||||
@@ -24,7 +24,7 @@ func (me *messageEvents) CheckMsg(smsg types.ChainMsg, hnd MsgHandler) CheckFunc
|
||||
return false, true, nil
|
||||
}
|
||||
|
||||
ml, err := me.cs.StateSearchMsg(ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true)
|
||||
ml, err := me.cs.StateSearchMsg(me.ctx, ts.Key(), msg.Cid(), stmgr.LookbackNoLimit, true)
|
||||
if err != nil {
|
||||
return false, true, xerrors.Errorf("getting receipt in CheckMsg: %w", err)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ package exchange
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
|
||||
var _ = xerrors.Errorf
|
||||
var _ = cid.Undef
|
||||
var _ = math.E
|
||||
var _ = sort.Sort
|
||||
|
||||
var lengthBufRequest = []byte{131}
|
||||
|
||||
@@ -151,20 +151,12 @@ func (c *client) doRequest(
|
||||
// errors. Peer penalization should happen here then, before returning, so
|
||||
// we can apply the correct penalties depending on the cause of the error.
|
||||
// FIXME: Add the `peer` as argument once we implement penalties.
|
||||
func (c *client) processResponse(req *Request, res *Response, tipsets []*types.TipSet) (r *validatedResponse, err error) {
|
||||
err = res.statusToError()
|
||||
func (c *client) processResponse(req *Request, res *Response, tipsets []*types.TipSet) (*validatedResponse, error) {
|
||||
err := res.statusToError()
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("status error: %s", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if rerr := recover(); rerr != nil {
|
||||
log.Errorf("process response error: %s", rerr)
|
||||
err = xerrors.Errorf("process response error: %s", rerr)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
options := parseOptions(req.Options)
|
||||
if options.noOptionsSet() {
|
||||
// Safety check: this shouldn't have been sent, and even if it did
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user