cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into fix/11-masyer

Browse Source
This commit is contained in:
Jennifer Wang 2021-07-27 20:01:52 -04:00
commit 9afb63d2e4
298 changed files with 18525 additions and 8543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
.circleci/config.yml
View File

@ -735,6 +735,45 @@ jobs:
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
publish-dockerhub:
description: publish to dockerhub
machine:
image: ubuntu-2004:202010-01
parameters:
tag:
type: string
default: latest
steps:
- checkout
- run:
name: dockerhub login
command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
- run:
name: docker build
command: |
docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
if [[ ! -z $CIRCLE_SHA1 ]]; then
docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
fi
if [[ ! -z $CIRCLE_TAG ]]; then
docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
fi
- run:
name: docker push
command: |
docker push filecoin/lotus:<< parameters.tag >>
docker push filecoin/lotus-all-in-one:<< parameters.tag >>
if [[ ! -z $CIRCLE_SHA1 ]]; then
docker push filecoin/lotus:$CIRCLE_SHA1
docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
fi
if [[ ! -z $CIRCLE_TAG ]]; then
docker push filecoin/lotus:$CIRCLE_TAG
docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
fi
workflows:
version: 2.1
@ -781,6 +820,11 @@ workflows:
suite: itest-deals_offline
target: "./itests/deals_offline_test.go"
- test:
name: test-itest-deals_padding
suite: itest-deals_padding
target: "./itests/deals_padding_test.go"
- test:
name: test-itest-deals_power
suite: itest-deals_power
@ -806,11 +850,21 @@ workflows:
suite: itest-gateway
target: "./itests/gateway_test.go"
- test:
name: test-itest-get_messages_in_ts
suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go"
- test:
name: test-itest-multisig
suite: itest-multisig
target: "./itests/multisig_test.go"
- test:
name: test-itest-nonce
suite: itest-nonce
target: "./itests/nonce_test.go"
- test:
name: test-itest-paych_api
suite: itest-paych_api
@ -831,6 +885,11 @@ workflows:
suite: itest-sector_finalize_early
target: "./itests/sector_finalize_early_test.go"
- test:
name: test-itest-sector_miner_collateral
suite: itest-sector_miner_collateral
target: "./itests/sector_miner_collateral_test.go"
- test:
name: test-itest-sector_pledge
suite: itest-sector_pledge
@ -1002,6 +1061,16 @@ workflows:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-dockerhub:
name: publish-dockerhub
tag: stable
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
nightly:
triggers:
@ -1015,3 +1084,6 @@ workflows:
- publish-snapcraft:
name: publish-snapcraft-nightly
channel: edge
- publish-dockerhub:
name: publish-dockerhub-nightly
tag: nightly

52
.circleci/template.yml
View File

@ -735,6 +735,45 @@ jobs:
- packer/build:
template: tools/packer/lotus.pkr.hcl
args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG"
publish-dockerhub:
description: publish to dockerhub
machine:
image: ubuntu-2004:202010-01
parameters:
tag:
type: string
default: latest
steps:
- checkout
- run:
name: dockerhub login
command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
- run:
name: docker build
command: |
docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
fi
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
fi
- run:
name: docker push
command: |
docker push filecoin/lotus:<< parameters.tag >>
docker push filecoin/lotus-all-in-one:<< parameters.tag >>
if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
docker push filecoin/lotus:$CIRCLE_SHA1
docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
fi
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
docker push filecoin/lotus:$CIRCLE_TAG
docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
fi
workflows:
version: 2.1
@ -887,6 +926,16 @@ workflows:
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- publish-dockerhub:
name: publish-dockerhub
tag: stable
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
nightly:
triggers:
@ -900,3 +949,6 @@ workflows:
- publish-snapcraft:
name: publish-snapcraft-nightly
channel: edge
- publish-dockerhub:
name: publish-dockerhub-nightly
tag: nightly

6
.github/CODEOWNERS vendored Normal file
View File

@ -0,0 +1,6 @@
# Reference
# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners
# Global owners
# Ensure maintainers team is a requested reviewer for non-draft PRs
* @filecoin-project/lotus-maintainers

33
.github/ISSUE_TEMPLATE/bug-report.md vendored
View File

@ -1,33 +0,0 @@
---
name: Bug Report
about: Create a report to help us improve
title: "[BUG] "
labels: hint/needs-triaging, kind/bug
assignees: ''
---
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
**Describe the bug**
A clear and concise description of what the bug is.
(If you are not sure what the bug is, try to figure it out via a [discussion](https://github.com/filecoin-project/lotus/discussions/new) first!
**Version (run `lotus version`):**
**To Reproduce**
Steps to reproduce the behavior:
1. Run '...'
2. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Logs**
Provide daemon/miner/worker logs, and goroutines(if available) for troubleshooting.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Additional context**
Add any other context about the problem here.

92
.github/ISSUE_TEMPLATE/bug_report.yml vendored Normal file
View File

@ -0,0 +1,92 @@
name: "Bug Report"
description: "File a bug report to help us improve"
labels: [need/triage, kind/bug]
body:
- type: checkboxes
attributes:
label: Checklist
description: Please check off the following boxes before continuing to file a bug report!
options:
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
required: true
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
required: true
- label: I did not make any code changes to lotus.
required: false
- type: dropdown
id: component-and-area
validations:
required: true
attributes:
label: Lotus component
description: Please select the lotus component you are filing a bug for
options:
- lotus daemon - chain sync
- lotus miner - mining and block production
- lotus miner/worker - sealing
- lotus miner - proving(WindowPoSt)
- lotus miner/market - storage deal
- lotus miner/market - retrieval deal
- lotus client
- lotus JSON-RPC API
- lotus message management (mpool)
- Other
- type: textarea
id: version
attributes:
label: Lotus Version
description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
placeholder: |
e.g.
Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0
Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
validations:
reuiqred: true
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doding when you experienced the bug?
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour?
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
* For proving issues, include the output of `lotus-miner proving` info.
* For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
render: bash
validations:
required: true
- type: textarea
id: extraInfo
attributes:
label: Logging Information
description: |
Please provide debug logs of the problem, remember you can get set log level control for:
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
render: bash
validations:
required: true
- type: textarea
id: RepoSteps
attributes:
label: Repo Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
render: bash
validations:
required: false

49
.github/ISSUE_TEMPLATE/deal-making-issues.md vendored
View File

@ -1,49 +0,0 @@
---
name: Deal Making Issues
about: Create a report for help with deal making failures.
title: "[Deal Making Issue]"
labels: hint/needs-triaging, area/markets
assignees: ''
---
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
Please provide all the information requested here to help us troubleshoot "deal making failed" issues.
If the information requested is missing, we will probably have to just ask you to provide it anyway,
before we can help debug.
**Basic Information**
Including information like, Are you the client or the miner? Is this a storage deal or a retrieval deal? Is it an offline deal?
**Describe the problem**
A brief description of the problem you encountered while trying to make a deal.
**Version**
The output of `lotus --version`.
**Setup**
You miner(if applicable) and daemon setup, i.e: What hardware do you use, how much ram and etc.
**To Reproduce**
Steps to reproduce the behavior:
1. Run '...'
2. See error
**Deal status**
The output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
**Lotus daemon and miner logs**
Please go through the logs of your daemon and miner(if applicable), and include screenshots of any error/warning-like messages you find.
Alternatively please upload full log files and share a link here
** Code modifications **
If you have modified parts of lotus, please describe which areas were modified,
and the scope of those modifications

44
.github/ISSUE_TEMPLATE/enhancement.yml vendored Normal file
View File

@ -0,0 +1,44 @@
name: Enhancement
description: Suggest an improvement to an existing lotus feature.
labels: [need/triage, kind/enhancement]
body:
- type: checkboxes
attributes:
label: Checklist
description: Please check off the following boxes before continuing to create an improvement suggestion!
options:
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated improvement to propose.
required: true
- type: dropdown
id: component
validations:
required: true
attributes:
label: Lotus component
description: Please select the lotus component you are propoing improvement for
options:
- lotus daemon - chain sync
- lotus miner - mining and block production
- lotus miner/worker - sealing
- lotus miner - proving(WindowPoSt)
- lotus miner/market - storage deal
- lotus miner/market - retrieval deal
- lotus client
- lotus JSON-RPC API
- lotus message management (mpool)
- Other
- type: textarea
id: request
attributes:
label: Improvement Suggestion
description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
validations:
required: true

20
.github/ISSUE_TEMPLATE/feature_request.md vendored
View File

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: "[Feature Request]"
labels: hint/needs-triaging
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

63
.github/ISSUE_TEMPLATE/feature_request.yml vendored Normal file
View File

@ -0,0 +1,63 @@
name: Feature request
description: Suggest an idea for lotus
labels: [need/triage, kind/feature]
body:
- type: checkboxes
attributes:
label: Checklist
description: Please check off the following boxes before continuing to create a new feature request!
options:
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated feature request to propose.
required: true
- type: dropdown
id: component
validations:
required: true
attributes:
label: Lotus component
description: Please select the lotus component you are requesting a new feature for
options:
- lotus daemon - chain sync
- lotus miner - mining and block production
- lotus miner/worker - sealing
- lotus miner - proving(WindowPoSt)
- lotus miner/market - storage deal
- lotus miner/market - retrieval deal
- lotus client
- lotus JSON-RPC API
- lotus message management (mpool)
- Other
- type: textarea
id: request
attributes:
label: What is the motivation behind this feature request? Is your feature request related to a problem? Please describe.
description: A clear and concise description of what the motivation or the problem is.
placeholder: Ex. I'm always frustrated when [...]
validations:
required: true
- type: textarea
id: solution
attributes:
label: Describe the solution you'd like
description: A clear and concise description of what you want to happen.
validations:
required: true
- type: textarea
id: alternates
attributes:
label: Describe alternatives you've considered
description: A clear and concise description of any alternative solutions or features you've considered.
validations:
required: false
- type: textarea
id: extra
attributes:
label: Additional context
description: Add any other context, design docs or screenshots about the feature request here.
validations:
required: false

91
.github/ISSUE_TEMPLATE/m1_bug_report_deal.yml vendored Normal file
View File

@ -0,0 +1,91 @@
name: "M1 Bug Report For Deal Making"
description: "File a bug report around deal making for the M1 releases"
labels: [need/triage, kind/bug, M1-release]
body:
- type: checkboxes
id: checklist
attributes:
label: Checklist
description: Please check off the following boxes before continuing to file a bug report!
options:
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
required: true
- label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
required: true
- label: I **am** reporting a bug around deal making. If not, create a [M1 Bug Report For Non Deal Making Issue](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_non_deal.yml).
required: true
- label: I have my log level set as instructed [here](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043678) and have logs available for troubleshooting.
required: true
- label: The deal is coming from one of the M1 clients(communitcated in the coordination slack channel).
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- type: dropdown
id: lotus-componets
validations:
required: true
attributes:
label: Lotus Component
description: Please select the lotus component you are filing a bug for
options:
- lotus miner market subsystem - storage deal
- lotus miner market subsystem - retrieval deal
- lotus miner - storage deal
- lotus miner - retrieval deal
- type: textarea
id: version
attributes:
label: Lotus Tag and Version
description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`.
validations:
reuiqred: true
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doding when you experienced the bug?
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour?
render: bash
validations:
required: true
- type: textarea
id: deal-status
attributes:
label: Deal Status
description: What's the status of the deal?
placeholder: |
Please share the output of `lotus-miner storage-deals|retrieval-deals list [-v]` commands for the deal(s) in question.
validations:
required: true
- type: textarea
id: data-transfer-status
attributes:
label: Data Transfer Status
description: What's the status of the data transfer?
placeholder: |
Please share the output of `lotus-miner data-transfers list -v` commands for the deal(s) in question.
validations:
required: true
- type: textarea
id: logging
attributes:
label: Logging Information
description: Please link to the whole of the miner logs on your side of the transaction. You can upload the logs to a [gist](https://gist.github.com).
validations:
required: true
- type: textarea
id: RepoSteps
attributes:
label: Repo Steps (optional)
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
render: bash
validations:
required: false

81
.github/ISSUE_TEMPLATE/m1_bug_report_non_deal.yml vendored Normal file
View File

@ -0,0 +1,81 @@
name: "M1 Bug Report For Non Deal Making Issue"
description: "File a bug report around non deal making issue for the M1 releases"
labels: [need/triage, kind/bug, M1-release]
body:
- type: checkboxes
id: checklist
attributes:
label: Checklist
description: Please check off the following boxes before continuing to file a bug report!
options:
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
required: true
- label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
required: true
- label: I am **not** reporting a bug around deal making. If yes, create a [M1 Bug Report For Deal Making](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_deal.yml).
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- type: dropdown
id: component-and-area
validations:
required: true
attributes:
label: Lotus component
description: Please select the lotus component you are filing a bug for
options:
- lotus daemon - chain sync **with** splitstore enabled
- lotus daemon - chain sync **without** splitstore enabled
- lotus miner - mining and block production
- lotus miner/worker - sealing
- lotus miner - proving(WindowPoSt)
- lotus client
- lotus JSON-RPC API
- lotus message management (mpool)
- Other
- type: textarea
id: version
attributes:
label: Lotus Tag and Version
description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`.
validations:
reuiqred: true
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doding when you experienced the bug?
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour?
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
* For proving issues, include the output of `lotus-miner proving` info.
render: bash
validations:
required: true
- type: textarea
id: extraInfo
attributes:
label: Logging Information
description: |
Please provide debug logs of the problem, remember you can get set log level control for:
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
render: bash
validations:
required: true
- type: textarea
id: RepoSteps
attributes:
label: Repo Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
render: bash
validations:
required: false

35
.github/ISSUE_TEMPLATE/mining-issues.md vendored
View File

@ -1,35 +0,0 @@
---
name: Mining Issues
about: Create a report for help with mining failures.
title: "[Mining Issue]"
labels: hint/needs-triaging, area/mining
assignees: ''
---
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
Please provide all the information requested here to help us troubleshoot "mining/WinningPoSt failed" issues.
If the information requested is missing, you may be asked you to provide it.
**Describe the problem**
A brief description of the problem you encountered while mining new blocks.
**Version**
The output of `lotus --version`.
**Setup**
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
**Lotus daemon and miner logs**
Please go through the logs of your daemon and miner, and include screenshots of any error/warning-like messages you find, highlighting the one has "winning post" in it.
Alternatively please upload full log files and share a link here
** Code modifications **
If you have modified parts of lotus, please describe which areas were modified,
and the scope of those modifications

46
.github/ISSUE_TEMPLATE/proving-issues.md vendored
View File

@ -1,46 +0,0 @@
---
name: Proving Issues
about: Create a report for help with proving failures.
title: "[Proving Issue]"
labels: area/proving, hint/needs-triaging
assignees: ''
---
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
Please provide all the information requested here to help us troubleshoot "proving/window PoSt failed" issues.
If the information requested is missing, we will probably have to just ask you to provide it anyway,
before we can help debug.
**Describe the problem**
A brief description of the problem you encountered while proving the storage.
**Version**
The output of `lotus --version`.
**Setup**
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
**Proving status**
The output of `lotus-miner proving` info.
**Lotus miner logs**
Please go through the logs of your miner, and include screenshots of any error-like messages you find, highlighting the one has "window post" in it.
Alternatively please upload full log files and share a link here
**Lotus miner diagnostic info**
Please collect the following diagnostic information, and share a link here
* lotus-miner diagnostic info `lotus-miner info all > allinfo.txt`
** Code modifications **
If you have modified parts of lotus, please describe which areas were modified,
and the scope of those modifications

50
.github/ISSUE_TEMPLATE/sealing-issues.md vendored
View File

@ -1,50 +0,0 @@
---
name: Sealing Issues
about: Create a report for help with sealing (commit) failures.
title: "[Sealing Issue]"
labels: hint/needs-triaging, area/sealing
assignees: ''
---
> Note: For security-related bugs/issues, please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
Please provide all the information requested here to help us troubleshoot "commit failed" issues.
If the information requested is missing, we will probably have to just ask you to provide it anyway,
before we can help debug.
**Describe the problem**
A brief description of the problem you encountered while sealing a sector.
**Version**
The output of `lotus --version`.
**Setup**
You miner and daemon setup, including what hardware do you use, your environment variable settings, how do you run your miner and worker, do you use GPU and etc.
**Commands**
Commands you ran.
**Sectors status**
The output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
**Lotus miner logs**
Please go through the logs of your miner, and include screenshots of any error-like messages you find.
Alternatively please upload full log files and share a link here
**Lotus miner diagnostic info**
Please collect the following diagnostic information, and share a link here
* lotus-miner diagnostic info `lotus-miner info all > allinfo`
** Code modifications **
If you have modified parts of lotus, please describe which areas were modified,
and the scope of those modifications

33
.github/workflows/stale.yml vendored Normal file
View File

@ -0,0 +1,33 @@
name: Close and mark stale issue
on:
schedule:
- cron: '0 12 * * *'
jobs:
stale:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/stale@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 24 hours.'
close-issue-message: 'This issue was closed because it is missing author input.'
stale-pr-message: 'Thank you for submitting the PR and contributing to lotus! Lotus maintainers need more of your input before merging it, please address the suggested changes or reply to the comments or this PR will be closed in 48 hours. You are always more than welcome to reopen the PR later as well!'
close-pr-message: 'This PR was closed because it is missing author input. Please feel free to reopen the PR when you get to it! Thank you for your interest in contributing to lotus!'
stale-issue-label: 'kind/stale'
stale-pr-label: 'kind/stale'
any-of-labels: 'need/author-input '
days-before-issue-stale: 3
days-before-issue-close: 1
days-before-pr-stale: 5
days-before-pr-close: 2
remove-stale-when-updated: true
enable-statistics: true

229
CHANGELOG.md
View File

@ -221,8 +221,230 @@ Contributors
| @zhoutian527 | 1 | +2/-2 | 1 |
| @ribasushi| 1 | +1/-1 | 1 |
||||||| 764fa9dae
# 1.11.0-rc1 / 2021-06-28
This is the first release candidate for the optional Lotus v1.11.0 release that introduces several months of bugfixes and feature development.
- github.com/filecoin-project/lotus:
- Lotus version 1.11.0
- gateway: Add support for Version method ([filecoin-project/lotus#6618](https://github.com/filecoin-project/lotus/pull/6618))
- Miner SimultaneousTransfers config ([filecoin-project/lotus#6612](https://github.com/filecoin-project/lotus/pull/6612))
- revamped integration test kit (aka. Operation Sparks Joy) ([filecoin-project/lotus#6329](https://github.com/filecoin-project/lotus/pull/6329))
- downgrade libp2p/go-libp2p-yamux to v0.5.1. ([filecoin-project/lotus#6605](https://github.com/filecoin-project/lotus/pull/6605))
- Fix wallet error messages ([filecoin-project/lotus#6594](https://github.com/filecoin-project/lotus/pull/6594))
- Fix CircleCI gen ([filecoin-project/lotus#6589](https://github.com/filecoin-project/lotus/pull/6589))
- Make query-ask CLI more graceful ([filecoin-project/lotus#6590](https://github.com/filecoin-project/lotus/pull/6590))
- ([filecoin-project/lotus#6406](https://github.com/filecoin-project/lotus/pull/6406))
- move with changed name ([filecoin-project/lotus#6587](https://github.com/filecoin-project/lotus/pull/6587))
- scale up sector expiration to avoid sector expire in batch-pre-commit waitting ([filecoin-project/lotus#6566](https://github.com/filecoin-project/lotus/pull/6566))
- Merge release branch into master ([filecoin-project/lotus#6583](https://github.com/filecoin-project/lotus/pull/6583))
- ([filecoin-project/lotus#6582](https://github.com/filecoin-project/lotus/pull/6582))
- fix circleci being out of sync. ([filecoin-project/lotus#6573](https://github.com/filecoin-project/lotus/pull/6573))
- dynamic circleci config for streamlining test execution ([filecoin-project/lotus#6561](https://github.com/filecoin-project/lotus/pull/6561))
- Merge 1.10 branch into master ([filecoin-project/lotus#6571](https://github.com/filecoin-project/lotus/pull/6571))
- Fix helptext ([filecoin-project/lotus#6560](https://github.com/filecoin-project/lotus/pull/6560))
- extern/storage: add ability to ignore worker resources when scheduling. ([filecoin-project/lotus#6542](https://github.com/filecoin-project/lotus/pull/6542))
- Merge 1.10 branch into master ([filecoin-project/lotus#6540](https://github.com/filecoin-project/lotus/pull/6540))
- Initial draft: basic build instructions on Readme ([filecoin-project/lotus#6498](https://github.com/filecoin-project/lotus/pull/6498))
- fix commit finalize failed ([filecoin-project/lotus#6521](https://github.com/filecoin-project/lotus/pull/6521))
- Dynamic Retrieval pricing ([filecoin-project/lotus#6175](https://github.com/filecoin-project/lotus/pull/6175))
- Fix soup ([filecoin-project/lotus#6501](https://github.com/filecoin-project/lotus/pull/6501))
- fix: pick the correct partitions-per-post limit ([filecoin-project/lotus#6502](https://github.com/filecoin-project/lotus/pull/6502))
- Fix the build
- Adjust various CLI display ratios to arbitrary precision ([filecoin-project/lotus#6309](https://github.com/filecoin-project/lotus/pull/6309))
- Add utils to use multisigs as miner owners ([filecoin-project/lotus#6490](https://github.com/filecoin-project/lotus/pull/6490))
- Test multicore SDR support ([filecoin-project/lotus#6479](https://github.com/filecoin-project/lotus/pull/6479))
- sealing: Fix restartSectors race ([filecoin-project/lotus#6495](https://github.com/filecoin-project/lotus/pull/6495))
- Merge 1.10 into master ([filecoin-project/lotus#6487](https://github.com/filecoin-project/lotus/pull/6487))
- Unit tests for sector batchers ([filecoin-project/lotus#6432](https://github.com/filecoin-project/lotus/pull/6432))
- Merge 1.10 changes into master ([filecoin-project/lotus#6466](https://github.com/filecoin-project/lotus/pull/6466))
- Update chain list with correct help instructions ([filecoin-project/lotus#6465](https://github.com/filecoin-project/lotus/pull/6465))
- clean failed sectors in batch commit ([filecoin-project/lotus#6451](https://github.com/filecoin-project/lotus/pull/6451))
- itests/kit: add guard to ensure imports from tests only. ([filecoin-project/lotus#6445](https://github.com/filecoin-project/lotus/pull/6445))
- consolidate integration tests into `itests` package; create test kit; cleanup ([filecoin-project/lotus#6311](https://github.com/filecoin-project/lotus/pull/6311))
- Remove rc changelog, compile the new changelog for final release only ([filecoin-project/lotus#6444](https://github.com/filecoin-project/lotus/pull/6444))
- updated configuration comments for docs ([filecoin-project/lotus#6440](https://github.com/filecoin-project/lotus/pull/6440))
- Set ntwk v13 HyperDrive Calibration upgrade epoch ([filecoin-project/lotus#6441](https://github.com/filecoin-project/lotus/pull/6441))
- Merge release/v1.10.10 into master ([filecoin-project/lotus#6439](https://github.com/filecoin-project/lotus/pull/6439))
- implement a command to export a car ([filecoin-project/lotus#6405](https://github.com/filecoin-project/lotus/pull/6405))
- Merge v1.10 release branch into master ([filecoin-project/lotus#6435](https://github.com/filecoin-project/lotus/pull/6435))
- Fee config for sector batching ([filecoin-project/lotus#6420](https://github.com/filecoin-project/lotus/pull/6420))
- Fix: correct the change of message size limit ([filecoin-project/lotus#6430](https://github.com/filecoin-project/lotus/pull/6430))
- UX: lotus state power CLI should fail if called with a not-miner ([filecoin-project/lotus#6425](https://github.com/filecoin-project/lotus/pull/6425))
- network reset friday
- Increase message size limit ([filecoin-project/lotus#6419](https://github.com/filecoin-project/lotus/pull/6419))
- polish(stmgr): define ExecMonitor for message application callback ([filecoin-project/lotus#6389](https://github.com/filecoin-project/lotus/pull/6389))
- upgrade testground action version ([filecoin-project/lotus#6403](https://github.com/filecoin-project/lotus/pull/6403))
- Fix logging of stringified CIDs double-encoded in hex ([filecoin-project/lotus#6413](https://github.com/filecoin-project/lotus/pull/6413))
- Update libp2p to 0.14.2 ([filecoin-project/lotus#6404](https://github.com/filecoin-project/lotus/pull/6404))
- Bypass task scheduler for reading unsealed pieces ([filecoin-project/lotus#6280](https://github.com/filecoin-project/lotus/pull/6280))
- testplans: lotus-soup: use default WPoStChallengeWindow ([filecoin-project/lotus#6400](https://github.com/filecoin-project/lotus/pull/6400))
- build snapcraft ([filecoin-project/lotus#6388](https://github.com/filecoin-project/lotus/pull/6388))
- Fix the doc errors of the sealing config funcs ([filecoin-project/lotus#6399](https://github.com/filecoin-project/lotus/pull/6399))
- Integration tests for offline deals ([filecoin-project/lotus#6081](https://github.com/filecoin-project/lotus/pull/6081))
- Fix success handling in Retreival ([filecoin-project/lotus#5921](https://github.com/filecoin-project/lotus/pull/5921))
- Fix some flaky tests ([filecoin-project/lotus#6397](https://github.com/filecoin-project/lotus/pull/6397))
- build appimage in CI ([filecoin-project/lotus#6384](https://github.com/filecoin-project/lotus/pull/6384))
- Add doc on gas balancing ([filecoin-project/lotus#6392](https://github.com/filecoin-project/lotus/pull/6392))
- Add a command to list retrievals ([filecoin-project/lotus#6337](https://github.com/filecoin-project/lotus/pull/6337))
- Add interop network ([filecoin-project/lotus#6387](https://github.com/filecoin-project/lotus/pull/6387))
- Network version 13 (v1.11) ([filecoin-project/lotus#6342](https://github.com/filecoin-project/lotus/pull/6342))
- Generate AppImage ([filecoin-project/lotus#6208](https://github.com/filecoin-project/lotus/pull/6208))
- lotus-gateway: add check command ([filecoin-project/lotus#6373](https://github.com/filecoin-project/lotus/pull/6373))
- Add a warning to the release issue template ([filecoin-project/lotus#6374](https://github.com/filecoin-project/lotus/pull/6374))
- update to markets-v1.4.0 ([filecoin-project/lotus#6369](https://github.com/filecoin-project/lotus/pull/6369))
- Add test for AddVerifiedClient ([filecoin-project/lotus#6317](https://github.com/filecoin-project/lotus/pull/6317))
- Typo fix in error message: "pubusb" -> "pubsub" ([filecoin-project/lotus#6365](https://github.com/filecoin-project/lotus/pull/6365))
- Improve the cli state call command ([filecoin-project/lotus#6226](https://github.com/filecoin-project/lotus/pull/6226))
- Upscale mineOne message to a WARN on unexpected ineligibility ([filecoin-project/lotus#6358](https://github.com/filecoin-project/lotus/pull/6358))
- storagefsm: Fix batch deal packing behavior ([filecoin-project/lotus#6041](https://github.com/filecoin-project/lotus/pull/6041))
- Remove few useless variable assignments ([filecoin-project/lotus#6359](https://github.com/filecoin-project/lotus/pull/6359))
- lotus-wallet: JWT Support ([filecoin-project/lotus#6360](https://github.com/filecoin-project/lotus/pull/6360))
- Reduce noise from 'peer has different genesis' messages ([filecoin-project/lotus#6357](https://github.com/filecoin-project/lotus/pull/6357))
- events: Fix handling of multiple matched events per epoch ([filecoin-project/lotus#6355](https://github.com/filecoin-project/lotus/pull/6355))
- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#6236](https://github.com/filecoin-project/lotus/pull/6236))
- Get current seal proof when necessary ([filecoin-project/lotus#6339](https://github.com/filecoin-project/lotus/pull/6339))
- Allow starting networks from arbitrary actor versions ([filecoin-project/lotus#6333](https://github.com/filecoin-project/lotus/pull/6333))
- Remove log line when tracing is not configured ([filecoin-project/lotus#6334](https://github.com/filecoin-project/lotus/pull/6334))
- Revert "Allow starting networks from arbitrary actor versions" ([filecoin-project/lotus#6330](https://github.com/filecoin-project/lotus/pull/6330))
- separate tracing environment variables ([filecoin-project/lotus#6323](https://github.com/filecoin-project/lotus/pull/6323))
- Allow starting networks from arbitrary actor versions ([filecoin-project/lotus#6305](https://github.com/filecoin-project/lotus/pull/6305))
- feat: log dispute rate ([filecoin-project/lotus#6322](https://github.com/filecoin-project/lotus/pull/6322))
- Use new actor tags ([filecoin-project/lotus#6291](https://github.com/filecoin-project/lotus/pull/6291))
- Fix logging around mineOne ([filecoin-project/lotus#6310](https://github.com/filecoin-project/lotus/pull/6310))
- Fix shell completions ([filecoin-project/lotus#6316](https://github.com/filecoin-project/lotus/pull/6316))
- Allow 8MB sectors in devnet ([filecoin-project/lotus#6312](https://github.com/filecoin-project/lotus/pull/6312))
- fix ticket expired ([filecoin-project/lotus#6304](https://github.com/filecoin-project/lotus/pull/6304))
- oh, snap! ([filecoin-project/lotus#6202](https://github.com/filecoin-project/lotus/pull/6202))
- Move verifreg shed utils to CLI ([filecoin-project/lotus#6135](https://github.com/filecoin-project/lotus/pull/6135))
- consider storiface.PathStorage when calculating storage requirements ([filecoin-project/lotus#6233](https://github.com/filecoin-project/lotus/pull/6233))
- `storage` module: add go docs and minor code quality refactors ([filecoin-project/lotus#6259](https://github.com/filecoin-project/lotus/pull/6259))
- Revert "chore: update go-libp2p" ([filecoin-project/lotus#6306](https://github.com/filecoin-project/lotus/pull/6306))
- Increase data transfer timeouts ([filecoin-project/lotus#6300](https://github.com/filecoin-project/lotus/pull/6300))
- gateway: spin off from cmd to package ([filecoin-project/lotus#6294](https://github.com/filecoin-project/lotus/pull/6294))
- Update to markets 1.3 ([filecoin-project/lotus#6149](https://github.com/filecoin-project/lotus/pull/6149))
- Add a shed util to count 64 GiB miner stats ([filecoin-project/lotus#6290](https://github.com/filecoin-project/lotus/pull/6290))
- Delete CODEOWNERS ([filecoin-project/lotus#6289](https://github.com/filecoin-project/lotus/pull/6289))
- Merge v1.9.0 to master ([filecoin-project/lotus#6275](https://github.com/filecoin-project/lotus/pull/6275))
- Backport 6200 to master ([filecoin-project/lotus#6272](https://github.com/filecoin-project/lotus/pull/6272))
- Introduce stateless offline dealflow, bypassing the FSM/deallists ([filecoin-project/lotus#5961](https://github.com/filecoin-project/lotus/pull/5961))
- chore: update go-libp2p ([filecoin-project/lotus#6231](https://github.com/filecoin-project/lotus/pull/6231))
- fix: wait-api should use GetAPI to acquire binary specific API ([filecoin-project/lotus#6246](https://github.com/filecoin-project/lotus/pull/6246))
- Update RELEASE_ISSUE_TEMPLATE.md
- fix(ci): Updates to lotus CI build process ([filecoin-project/lotus#6256](https://github.com/filecoin-project/lotus/pull/6256))
- add flags to control gateway lookback parameters ([filecoin-project/lotus#6247](https://github.com/filecoin-project/lotus/pull/6247))
- Feat/nerpa v4 ([filecoin-project/lotus#6248](https://github.com/filecoin-project/lotus/pull/6248))
- chore(ci): Enable build on RC tags ([filecoin-project/lotus#6238](https://github.com/filecoin-project/lotus/pull/6238))
- Transplant some useful commands to lotus-shed actor ([filecoin-project/lotus#5913](https://github.com/filecoin-project/lotus/pull/5913))
- wip actor wrapper codegen ([filecoin-project/lotus#6108](https://github.com/filecoin-project/lotus/pull/6108))
- Robust message management ([filecoin-project/lotus#5822](https://github.com/filecoin-project/lotus/pull/5822))
- Add a shed util to count miners by post type ([filecoin-project/lotus#6169](https://github.com/filecoin-project/lotus/pull/6169))
- Introduce a release issue template ([filecoin-project/lotus#5826](https://github.com/filecoin-project/lotus/pull/5826))
- cron-wc ([filecoin-project/lotus#6178](https://github.com/filecoin-project/lotus/pull/6178))
- This is a 1:1 forward-port of PR#6183 from 1.9.x to master ([filecoin-project/lotus#6196](https://github.com/filecoin-project/lotus/pull/6196))
- Allow creation of state tree v3s ([filecoin-project/lotus#6167](https://github.com/filecoin-project/lotus/pull/6167))
- drand: fix beacon cache ([filecoin-project/lotus#6164](https://github.com/filecoin-project/lotus/pull/6164))
- Update cli gen ([filecoin-project/lotus#6155](https://github.com/filecoin-project/lotus/pull/6155))
- mpool: Cleanup pre-nv12 selection logic ([filecoin-project/lotus#6148](https://github.com/filecoin-project/lotus/pull/6148))
- Update ffi to proofs v7 ([filecoin-project/lotus#6150](https://github.com/filecoin-project/lotus/pull/6150))
- Generate CLI docs ([filecoin-project/lotus#6145](https://github.com/filecoin-project/lotus/pull/6145))
- feat: allow checkpointing to forks ([filecoin-project/lotus#6107](https://github.com/filecoin-project/lotus/pull/6107))
- attempt to do better padding on pieces being written into sectors ([filecoin-project/lotus#5988](https://github.com/filecoin-project/lotus/pull/5988))
- remove duplicate ask and calculate ping before lock ([filecoin-project/lotus#5968](https://github.com/filecoin-project/lotus/pull/5968))
- Add a command to get the fees of a deal ([filecoin-project/lotus#5307](https://github.com/filecoin-project/lotus/pull/5307))
- flaky tests improvement: separate TestBatchDealInput from TestAPIDealFlow ([filecoin-project/lotus#6141](https://github.com/filecoin-project/lotus/pull/6141))
- Testground checks on push ([filecoin-project/lotus#5887](https://github.com/filecoin-project/lotus/pull/5887))
- Add a CLI tool for miner proving deadline ([filecoin-project/lotus#6132](https://github.com/filecoin-project/lotus/pull/6132))
- Use EmptyTSK where appropriate ([filecoin-project/lotus#6134](https://github.com/filecoin-project/lotus/pull/6134))
- fix: use a consistent tipset in commands ([filecoin-project/lotus#6142](https://github.com/filecoin-project/lotus/pull/6142))
- go mod tidy for lotus-soup testplans ([filecoin-project/lotus#6124](https://github.com/filecoin-project/lotus/pull/6124))
- fix testground payment channel tests: use 1 miner ([filecoin-project/lotus#6126](https://github.com/filecoin-project/lotus/pull/6126))
- fix: use the parent state when listing actors ([filecoin-project/lotus#6143](https://github.com/filecoin-project/lotus/pull/6143))
- Speed up StateListMessages in some cases ([filecoin-project/lotus#6007](https://github.com/filecoin-project/lotus/pull/6007))
- Return total power when GetPowerRaw doesn't find miner claim ([filecoin-project/lotus#4938](https://github.com/filecoin-project/lotus/pull/4938))
- fix(splitstore): fix a panic on revert-only head changes ([filecoin-project/lotus#6133](https://github.com/filecoin-project/lotus/pull/6133))
- shed: command to list duplicate messages in tipsets (steb) ([filecoin-project/lotus#5847](https://github.com/filecoin-project/lotus/pull/5847))
- upgrade `lotus-soup` testplans and reduce deals concurrency to a single miner ([filecoin-project/lotus#6122](https://github.com/filecoin-project/lotus/pull/6122))
- Merge releases (1.8.0) into master ([filecoin-project/lotus#6118](https://github.com/filecoin-project/lotus/pull/6118))
- github.com/filecoin-project/go-commp-utils (v0.1.0 -> v0.1.1-0.20210427191551-70bf140d31c7):
- add a padding helper function ([filecoin-project/go-commp-utils#3](https://github.com/filecoin-project/go-commp-utils/pull/3))
- github.com/filecoin-project/go-data-transfer (v1.4.3 -> v1.6.0):
- release: v1.6.0
- fix: option to disable accept and complete timeouts
- fix: disable restart ack timeout
- release: v1.5.0
- Add isRestart param to validators (#197) ([filecoin-project/go-data-transfer#197](https://github.com/filecoin-project/go-data-transfer/pull/197))
- fix: flaky TestChannelMonitorAutoRestart (#198) ([filecoin-project/go-data-transfer#198](https://github.com/filecoin-project/go-data-transfer/pull/198))
- Channel monitor watches for errors instead of measuring data rate (#190) ([filecoin-project/go-data-transfer#190](https://github.com/filecoin-project/go-data-transfer/pull/190))
- fix: prevent concurrent restarts for same channel (#195) ([filecoin-project/go-data-transfer#195](https://github.com/filecoin-project/go-data-transfer/pull/195))
- fix: channel state machine event handling (#194) ([filecoin-project/go-data-transfer#194](https://github.com/filecoin-project/go-data-transfer/pull/194))
- Dont double count data sent (#185) ([filecoin-project/go-data-transfer#185](https://github.com/filecoin-project/go-data-transfer/pull/185))
- release: v1.4.3 (#189) ([filecoin-project/go-data-transfer#189](https://github.com/filecoin-project/go-data-transfer/pull/189))
- github.com/filecoin-project/go-fil-markets (v1.2.5 -> v1.5.0):
- release: v1.5.0
- Dynamic Retrieval Pricing (#542) ([filecoin-project/go-fil-markets#542](https://github.com/filecoin-project/go-fil-markets/pull/542))
- release: v1.4.0 (#551) ([filecoin-project/go-fil-markets#551](https://github.com/filecoin-project/go-fil-markets/pull/551))
- Update to go data transfer v1.6.0 (#550) ([filecoin-project/go-fil-markets#550](https://github.com/filecoin-project/go-fil-markets/pull/550))
- fix first make error (#548) ([filecoin-project/go-fil-markets#548](https://github.com/filecoin-project/go-fil-markets/pull/548))
- release: v1.3.0 (#544) ([filecoin-project/go-fil-markets#544](https://github.com/filecoin-project/go-fil-markets/pull/544))
- fix restarts during data transfer for a retrieval deal (#540) ([filecoin-project/go-fil-markets#540](https://github.com/filecoin-project/go-fil-markets/pull/540))
- Test Retrieval for offline deals (#541) ([filecoin-project/go-fil-markets#541](https://github.com/filecoin-project/go-fil-markets/pull/541))
- Allow anonymous submodule checkout (#535) ([filecoin-project/go-fil-markets#535](https://github.com/filecoin-project/go-fil-markets/pull/535))
- github.com/filecoin-project/specs-actors (v0.9.13 -> v0.9.14):
- Set ConsensusMinerMinPower to 10 TiB (#1427) ([filecoin-project/specs-actors#1427](https://github.com/filecoin-project/specs-actors/pull/1427))
- github.com/filecoin-project/specs-actors/v2 (v2.3.5-0.20210114162132-5b58b773f4fb -> v2.3.5):
- Set ConsensusMinerMinPower to 10 TiB (#1428) ([filecoin-project/specs-actors#1428](https://github.com/filecoin-project/specs-actors/pull/1428))
- v2 VM satisfies SimVM interface (#1355) ([filecoin-project/specs-actors#1355](https://github.com/filecoin-project/specs-actors/pull/1355))
- github.com/filecoin-project/specs-actors/v3 (v3.1.0 -> v3.1.1):
- Set ConsensusMinerMinPower to 10 TiB for all PoStProofPolicies (#1429) ([filecoin-project/specs-actors#1429](https://github.com/filecoin-project/specs-actors/pull/1429))
- github.com/filecoin-project/specs-actors/v4 (v4.0.0 -> v4.0.1):
- Set ConsensusMinerMinPower to 10 TiB for all PoStProofPolicies (#1430) ([filecoin-project/specs-actors#1430](https://github.com/filecoin-project/specs-actors/pull/1430))
Contributors
| Contributor | Commits | Lines ± | Files Changed |
|-------------|---------|---------|---------------|
| Raúl Kripalani | 118 | +11972/-10860 | 472 |
| Łukasz Magiera | 65 | +10824/-4158 | 353 |
| aarshkshah1992 | 59 | +8057/-3355 | 224 |
| Aayush Rajasekaran | 41 | +8786/-1691 | 331 |
| Steven Allen | 106 | +7653/-2718 | 273 |
| dirkmc | 11 | +2580/-1371 | 77 |
| Dirk McCormick | 39 | +1865/-1194 | 79 |
| Jakub Sztandera | 19 | +1973/-485 | 81 |
| vyzo | 4 | +1748/-330 | 50 |
| Aarsh Shah | 5 | +1462/-213 | 27 |
| Cory Schwartz | 35 | +568/-206 | 59 |
| chadwick2143 | 3 | +739/-1 | 4 |
| Peter Rabbitson | 21 | +487/-164 | 36 |
| hannahhoward | 5 | +544/-5 | 19 |
| Jennifer Wang | 8 | +206/-172 | 17 |
| frrist | 1 | +137/-88 | 7 |
| Travis Person | 3 | +175/-6 | 7 |
| Alex Wade | 1 | +48/-129 | 1 |
| whyrusleeping | 8 | +161/-13 | 11 |
| lotus | 1 | +114/-46 | 1 |
| Anton Evangelatov | 8 | +107/-53 | 20 |
| Rjan | 4 | +115/-33 | 4 |
| ZenGround0 | 3 | +114/-1 | 4 |
| Aloxaf | 1 | +43/-61 | 7 |
| yaohcn | 4 | +89/-9 | 5 |
| mitchellsoo | 1 | +51/-0 | 1 |
| Mike Greenberg | 3 | +28/-18 | 4 |
| Jennifer | 6 | +9/-14 | 6 |
| Frank | 2 | +11/-10 | 2 |
| wangchao | 3 | +5/-4 | 4 |
| Steve Loeppky | 1 | +7/-1 | 1 |
| Lion | 1 | +4/-2 | 1 |
| Mimir | 1 | +2/-2 | 1 |
| raulk | 1 | +1/-1 | 1 |
| Jack Yao | 1 | +1/-1 | 1 |
| IPFSUnion | 1 | +1/-1 | 1 |
||||||||| 764fa9dae
=========
=======
>>>>>>> master
# 1.10.1 / 2021-07-05
This is an optional but **highly recommended** release of Lotus for lotus miners that has many bug fixes and improvements based on the feedback we got from the community since HyperDrive.
@ -256,7 +478,12 @@ Contributors
| @ribasushi| 1 | +1/-1 | 1 |
<<<<<<< HEAD
>>>>>>> releases
||||||| merged common ancestors
>>>>>>>>> Temporary merge branch 2
=======
>>>>>>> master
# 1.10.0 / 2021-06-23
This is a mandatory release of Lotus that introduces Filecoin network v13, codenamed the HyperDrive upgrade. The

162
Dockerfile.lotus
View File

@ -36,7 +36,7 @@ WORKDIR /opt/filecoin
ARG RUSTFLAGS=""
ARG GOFLAGS=""
RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats
RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
FROM ubuntu:20.04 AS base
@ -56,19 +56,173 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
RUN useradd -r -u 532 -U fc
###
FROM base AS lotus
MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
COPY scripts/docker-lotus-entrypoint.sh /
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
ENV DOCKER_LOTUS_IMPORT_WALLET ""
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus
VOLUME /var/tmp/filecoin-proof-parameters
USER fc
ENTRYPOINT ["/usr/local/bin/lotus"]
EXPOSE 1234
ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
CMD ["-help"]
###
FROM base AS lotus-wallet
MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
ENV WALLET_PATH /var/lib/lotus-wallet
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
RUN mkdir /var/lib/lotus-wallet
RUN chown fc: /var/lib/lotus-wallet
VOLUME /var/lib/lotus-wallet
USER fc
EXPOSE 1777
ENTRYPOINT ["/usr/local/bin/lotus-wallet"]
CMD ["-help"]
###
FROM base AS lotus-gateway
MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
USER fc
EXPOSE 1234
ENTRYPOINT ["/usr/local/bin/lotus-gateway"]
CMD ["-help"]
###
FROM base AS lotus-miner
MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY scripts/docker-lotus-miner-entrypoint.sh /
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV DOCKER_LOTUS_MINER_INIT true
RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus-miner
VOLUME /var/tmp/filecoin-proof-parameters
USER fc
EXPOSE 2345
ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"]
CMD ["-help"]
###
FROM base AS lotus-worker
MAINTAINER Lotus Development Team
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
RUN mkdir /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-worker
VOLUME /var/lib/lotus-worker
USER fc
EXPOSE 3456
ENTRYPOINT ["/usr/local/bin/lotus-worker"]
CMD ["-help"]
###
from base as lotus-all-in-one
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
ENV LOTUS_JAEGER_AGENT_PORT 6831
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
ENV WALLET_PATH /var/lib/lotus-wallet
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
ENV DOCKER_LOTUS_MINER_INIT true
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
RUN mkdir /var/tmp/filecoin-proof-parameters
RUN mkdir /var/lib/lotus
RUN mkdir /var/lib/lotus-miner
RUN mkdir /var/lib/lotus-worker
RUN mkdir /var/lib/lotus-wallet
RUN chown fc: /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus
RUN chown fc: /var/lib/lotus-miner
RUN chown fc: /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-wallet
VOLUME /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus
VOLUME /var/lib/lotus-miner
VOLUME /var/lib/lotus-worker
VOLUME /var/lib/lotus-wallet
EXPOSE 1234
EXPOSE 2345
EXPOSE 3456
EXPOSE 1777

10
Makefile
View File

@ -92,7 +92,7 @@ BINS+=lotus
lotus-miner: $(BUILD_DEPS)
rm -f lotus-miner
go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-storage-miner
go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
.PHONY: lotus-miner
BINS+=lotus-miner
@ -131,6 +131,9 @@ install-miner:
install-worker:
install -C ./lotus-worker /usr/local/bin/lotus-worker
install-app:
install -C ./$(APP) /usr/local/bin/$(APP)
# TOOLS
lotus-seed: $(BUILD_DEPS)
@ -333,6 +336,9 @@ api-gen:
goimports -w api
.PHONY: api-gen
cfgdoc-gen:
go run ./node/config/cfgdocgen > ./node/config/doc_gen.go
appimage: lotus
rm -rf appimage-builder-cache || true
rm AppDir/io.filecoin.lotus.desktop || true
@ -370,7 +376,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
gen: actors-gen type-gen method-gen docsgen api-gen circleci
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen

2
README.md
View File

@ -18,6 +18,8 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
## Building & Documentation
> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
## Reporting a Vulnerability

55
api/api_common.go
View File

@ -4,15 +4,11 @@ import (
"context"
"fmt"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/google/uuid"
"github.com/filecoin-project/go-jsonrpc/auth"
metrics "github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
apitypes "github.com/filecoin-project/lotus/api/types"
)
// MODIFYING THE API INTERFACE
@ -27,55 +23,23 @@ import (
// * Generate openrpc blobs
type Common interface {
// MethodGroup: Auth
AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) //perm:admin
// MethodGroup: Net
// MethodGroup: Log
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
NetConnect(context.Context, peer.AddrInfo) error //perm:write
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
NetDisconnect(context.Context, peer.ID) error //perm:write
NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) //perm:read
NetPubsubScores(context.Context) ([]PubsubScore, error) //perm:read
NetAutoNatStatus(context.Context) (NatInfo, error) //perm:read
NetAgentVersion(ctx context.Context, p peer.ID) (string, error) //perm:read
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error) //perm:read
// NetBandwidthStats returns statistics about the nodes total bandwidth
// usage and current rate across all peers and protocols.
NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
// usage and current rate per peer
NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
// usage and current rate per protocol
NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
// ConnectionGater API
NetBlockAdd(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
LogList(context.Context) ([]string, error) //perm:write
LogSetLevel(context.Context, string, string) error //perm:write
// MethodGroup: Common
// Discover returns an OpenRPC document describing an RPC API.
Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error) //perm:read
// Version provides information about API provider
Version(context.Context) (APIVersion, error) //perm:read
LogList(context.Context) ([]string, error) //perm:write
LogSetLevel(context.Context, string, string) error //perm:write
// Discover returns an OpenRPC document describing an RPC API.
Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
// trigger graceful shutdown
Shutdown(context.Context) error //perm:admin
@ -105,8 +69,3 @@ type APIVersion struct {
func (v APIVersion) String() string {
return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
}
type NatInfo struct {
Reachability network.Reachability
PublicAddr string
}

11
api/api_full.go
View File

@ -58,6 +58,7 @@ const LookbackNoLimit = abi.ChainEpoch(-1)
// FullNode API is a low-level interface to the Filecoin network full node
type FullNode interface {
Common
Net
// MethodGroup: Chain
// The Chain method group contains methods for interacting with the
@ -104,6 +105,9 @@ type FullNode interface {
// specified block.
ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]Message, error) //perm:read
// ChainGetMessagesInTipset returns message stores in current tipset
ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]Message, error) //perm:read
// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
// If there are no blocks at the specified epoch, a tipset at an earlier epoch
// will be returned.
@ -160,6 +164,13 @@ type FullNode interface {
// If oldmsgskip is set, messages from before the requested roots are also not included.
ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
// if supported by the underlying implementation.
ChainCheckBlockstore(context.Context) error //perm:admin
// ChainBlockstoreInfo returns some basic information about the blockstore
ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
// MethodGroup: Beacon
// The Beacon method group contains methods for interacting with the random beacon (DRAND)

1
api/api_gateway.go
View File

@ -45,6 +45,7 @@ type Gateway interface {
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)

66
api/api_net.go Normal file
View File

@ -0,0 +1,66 @@
package api
import (
"context"
metrics "github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/protocol"
)
// MODIFYING THE API INTERFACE
//
// When adding / changing methods in this file:
// * Do the change here
// * Adjust implementation in `node/impl/`
// * Run `make gen` - this will:
// * Generate proxy structs
// * Generate mocks
// * Generate markdown docs
// * Generate openrpc blobs
type Net interface {
// MethodGroup: Net
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
NetConnect(context.Context, peer.AddrInfo) error //perm:write
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
NetDisconnect(context.Context, peer.ID) error //perm:write
NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error) //perm:read
NetPubsubScores(context.Context) ([]PubsubScore, error) //perm:read
NetAutoNatStatus(context.Context) (NatInfo, error) //perm:read
NetAgentVersion(ctx context.Context, p peer.ID) (string, error) //perm:read
NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error) //perm:read
// NetBandwidthStats returns statistics about the nodes total bandwidth
// usage and current rate across all peers and protocols.
NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
// usage and current rate per peer
NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
// usage and current rate per protocol
NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
// ConnectionGater API
NetBlockAdd(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error) //perm:read
}
type CommonNet interface {
Common
Net
}
type NatInfo struct {
Reachability network.Reachability
PublicAddr string
}

40
api/api_storage.go
View File

@ -41,6 +41,7 @@ import (
// StorageMiner is a low-level interface to the Filecoin network storage miner node
type StorageMiner interface {
Common
Net
ActorAddress(context.Context) (address.Address, error) //perm:read
@ -55,6 +56,13 @@ type StorageMiner interface {
// Get the status of a given sector by ID
SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read
// Add piece to an open sector. If no sectors with enough space are open,
// either a new sector will be created, or this call will block until more
// sectors can be created.
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
// List all staged sectors
SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@ -135,8 +143,8 @@ type StorageMiner interface {
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
StorageLocal(ctx context.Context) (map[stores.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
@ -279,15 +287,17 @@ type AddrUse int
const (
PreCommitAddr AddrUse = iota
CommitAddr
DealPublishAddr
PoStAddr
TerminateSectorsAddr
)
type AddressConfig struct {
PreCommitControl []address.Address
CommitControl []address.Address
TerminateControl []address.Address
PreCommitControl []address.Address
CommitControl []address.Address
TerminateControl []address.Address
DealPublishControl []address.Address
DisableOwnerFallback bool
DisableWorkerFallback bool
@ -300,3 +310,25 @@ type PendingDealInfo struct {
PublishPeriodStart time.Time
PublishPeriod time.Duration
}
type SectorOffset struct {
Sector abi.SectorNumber
Offset abi.PaddedPieceSize
}
// DealInfo is a tuple of deal identity and its schedule
type PieceDealInfo struct {
PublishCid *cid.Cid
DealID abi.DealID
DealProposal *market.DealProposal
DealSchedule DealSchedule
KeepUnsealed bool
}
// DealSchedule communicates the time interval of a storage deal. The deal must
// appear in a sealed (proven) sector no later than StartEpoch, otherwise it
// is invalid.
type DealSchedule struct {
StartEpoch abi.ChainEpoch
EndEpoch abi.ChainEpoch
}

379
api/cbor_gen.go
View File

@ -8,6 +8,7 @@ import (
"sort"
abi "github.com/filecoin-project/go-state-types/abi"
market "github.com/filecoin-project/specs-actors/actors/builtin/market"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
@ -738,3 +739,381 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
return nil
}
func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{165}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.PublishCid (cid.Cid) (struct)
if len("PublishCid") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"PublishCid\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("PublishCid")); err != nil {
return err
}
if t.PublishCid == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
}
}
// t.DealID (abi.DealID) (uint64)
if len("DealID") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"DealID\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealID")); err != nil {
return err
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
return err
}
// t.DealProposal (market.DealProposal) (struct)
if len("DealProposal") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"DealProposal\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealProposal")); err != nil {
return err
}
if err := t.DealProposal.MarshalCBOR(w); err != nil {
return err
}
// t.DealSchedule (api.DealSchedule) (struct)
if len("DealSchedule") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
return err
}
if err := t.DealSchedule.MarshalCBOR(w); err != nil {
return err
}
// t.KeepUnsealed (bool) (bool)
if len("KeepUnsealed") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
return err
}
if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil {
return err
}
return nil
}
func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
*t = PieceDealInfo{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
if extra > cbg.MaxLength {
return fmt.Errorf("PieceDealInfo: map struct too large (%d)", extra)
}
var name string
n := extra
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
name = string(sval)
}
switch name {
// t.PublishCid (cid.Cid) (struct)
case "PublishCid":
{
b, err := br.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
}
t.PublishCid = &c
}
}
// t.DealID (abi.DealID) (uint64)
case "DealID":
{
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = abi.DealID(extra)
}
// t.DealProposal (market.DealProposal) (struct)
case "DealProposal":
{
b, err := br.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := br.UnreadByte(); err != nil {
return err
}
t.DealProposal = new(market.DealProposal)
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
}
}
}
// t.DealSchedule (api.DealSchedule) (struct)
case "DealSchedule":
{
if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
}
}
// t.KeepUnsealed (bool) (bool)
case "KeepUnsealed":
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajOther {
return fmt.Errorf("booleans must be major type 7")
}
switch extra {
case 20:
t.KeepUnsealed = false
case 21:
t.KeepUnsealed = true
default:
return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra)
}
default:
// Field doesn't exist on this type, so ignore it
cbg.ScanForLinks(r, func(cid.Cid) {})
}
}
return nil
}
func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{162}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.StartEpoch (abi.ChainEpoch) (int64)
if len("StartEpoch") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
return err
}
if t.StartEpoch >= 0 {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
return err
}
} else {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
return err
}
}
// t.EndEpoch (abi.ChainEpoch) (int64)
if len("EndEpoch") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
return err
}
if t.EndEpoch >= 0 {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
return err
}
} else {
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
return err
}
}
return nil
}
func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
*t = DealSchedule{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
if err != nil {
return err
}
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
if extra > cbg.MaxLength {
return fmt.Errorf("DealSchedule: map struct too large (%d)", extra)
}
var name string
n := extra
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadStringBuf(br, scratch)
if err != nil {
return err
}
name = string(sval)
}
switch name {
// t.StartEpoch (abi.ChainEpoch) (int64)
case "StartEpoch":
{
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
}
switch maj {
case cbg.MajUnsignedInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 positive overflow")
}
case cbg.MajNegativeInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 negative oveflow")
}
extraI = -1 - extraI
default:
return fmt.Errorf("wrong type for int64 field: %d", maj)
}
t.StartEpoch = abi.ChainEpoch(extraI)
}
// t.EndEpoch (abi.ChainEpoch) (int64)
case "EndEpoch":
{
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
var extraI int64
if err != nil {
return err
}
switch maj {
case cbg.MajUnsignedInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 positive overflow")
}
case cbg.MajNegativeInt:
extraI = int64(extra)
if extraI < 0 {
return fmt.Errorf("int64 negative oveflow")
}
extraI = -1 - extraI
default:
return fmt.Errorf("wrong type for int64 field: %d", maj)
}
t.EndEpoch = abi.ChainEpoch(extraI)
}
default:
// Field doesn't exist on this type, so ignore it
cbg.ScanForLinks(r, func(cid.Cid) {})
}
}
return nil
}

85
api/client/client.go
View File

@ -16,14 +16,10 @@ import (
)
// NewCommonRPCV0 creates a new http jsonrpc client.
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
var res v0api.CommonStruct
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
var res v0api.CommonNetStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.Internal,
},
requestHeader,
)
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
@ -31,11 +27,9 @@ func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header)
// NewFullNodeRPCV0 creates a new http jsonrpc client.
func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.FullNode, jsonrpc.ClientCloser, error) {
var res v0api.FullNodeStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.CommonStruct.Internal,
&res.Internal,
}, requestHeader)
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
@ -44,51 +38,56 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
var res v1api.FullNodeStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.CommonStruct.Internal,
&res.Internal,
}, requestHeader)
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
func getPushUrl(addr string) (string, error) {
pushUrl, err := url.Parse(addr)
if err != nil {
return "", err
}
switch pushUrl.Scheme {
case "ws":
pushUrl.Scheme = "http"
case "wss":
pushUrl.Scheme = "https"
}
///rpc/v0 -> /rpc/streams/v0/push
pushUrl.Path = path.Join(pushUrl.Path, "../streams/v0/push")
return pushUrl.String(), nil
}
// NewStorageMinerRPCV0 creates a new http jsonrpc client for miner
func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.StorageMiner, jsonrpc.ClientCloser, error) {
pushUrl, err := getPushUrl(addr)
if err != nil {
return nil, nil, err
}
var res v0api.StorageMinerStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.CommonStruct.Internal,
&res.Internal,
},
requestHeader,
opts...,
)
api.GetInternalStructs(&res), requestHeader,
append([]jsonrpc.Option{
rpcenc.ReaderParamEncoder(pushUrl),
}, opts...)...)
return &res, closer, err
}
func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Worker, jsonrpc.ClientCloser, error) {
u, err := url.Parse(addr)
func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header) (v0api.Worker, jsonrpc.ClientCloser, error) {
pushUrl, err := getPushUrl(addr)
if err != nil {
return nil, nil, err
}
switch u.Scheme {
case "ws":
u.Scheme = "http"
case "wss":
u.Scheme = "https"
}
///rpc/v0 -> /rpc/streams/v0/push
u.Path = path.Join(u.Path, "../streams/v0/push")
var res api.WorkerStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.Internal,
},
api.GetInternalStructs(&res),
requestHeader,
rpcenc.ReaderParamEncoder(u.String()),
rpcenc.ReaderParamEncoder(pushUrl),
jsonrpc.WithNoReconnect(),
jsonrpc.WithTimeout(30*time.Second),
)
@ -100,9 +99,7 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.Gateway, jsonrpc.ClientCloser, error) {
var res api.GatewayStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.Internal,
},
api.GetInternalStructs(&res),
requestHeader,
opts...,
)
@ -114,9 +111,7 @@ func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header
func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (v0api.Gateway, jsonrpc.ClientCloser, error) {
var res v0api.GatewayStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.Internal,
},
api.GetInternalStructs(&res),
requestHeader,
opts...,
)
@ -127,9 +122,7 @@ func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header
func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.Wallet, jsonrpc.ClientCloser, error) {
var res api.WalletStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
[]interface{}{
&res.Internal,
},
api.GetInternalStructs(&res),
requestHeader,
)

2
api/docgen-openrpc/cmd/docgen_openrpc.go
View File

@ -34,7 +34,7 @@ func main() {
doc := docgen_openrpc.NewLotusOpenRPCDocument(Comments, GroupDocs)
i, _, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
i, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
doc.RegisterReceiverName("Filecoin", i)
out, err := doc.Discover()

17
api/docgen/cmd/docgen.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strings"
@ -15,7 +16,7 @@ func main() {
groups := make(map[string]*docgen.MethodGroup)
_, t, permStruct, commonPermStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
_, t, permStruct := docgen.GetAPIType(os.Args[2], os.Args[3])
for i := 0; i < t.NumMethod(); i++ {
m := t.Method(i)
@ -88,13 +89,17 @@ func main() {
fmt.Printf("### %s\n", m.Name)
fmt.Printf("%s\n\n", m.Comment)
meth, ok := permStruct.FieldByName(m.Name)
if !ok {
meth, ok = commonPermStruct.FieldByName(m.Name)
if !ok {
panic("no perms for method: " + m.Name)
var meth reflect.StructField
var ok bool
for _, ps := range permStruct {
meth, ok = ps.FieldByName(m.Name)
if ok {
break
}
}
if !ok {
panic("no perms for method: " + m.Name)
}
perms := meth.Tag.Get("perm")

21
api/docgen/docgen.go
View File

@ -266,25 +266,27 @@ func init() {
addExample(map[string]interface{}{"abc": 123})
}
func GetAPIType(name, pkg string) (i interface{}, t, permStruct, commonPermStruct reflect.Type) {
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
switch pkg {
case "api": // latest
switch name {
case "FullNode":
i = &api.FullNodeStruct{}
t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
permStruct = reflect.TypeOf(api.FullNodeStruct{}.Internal)
commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
permStruct = append(permStruct, reflect.TypeOf(api.FullNodeStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
case "StorageMiner":
i = &api.StorageMinerStruct{}
t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
permStruct = reflect.TypeOf(api.StorageMinerStruct{}.Internal)
commonPermStruct = reflect.TypeOf(api.CommonStruct{}.Internal)
permStruct = append(permStruct, reflect.TypeOf(api.StorageMinerStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(api.CommonStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(api.NetStruct{}.Internal))
case "Worker":
i = &api.WorkerStruct{}
t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
permStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
commonPermStruct = reflect.TypeOf(api.WorkerStruct{}.Internal)
permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
default:
panic("unknown type")
}
@ -293,8 +295,9 @@ func GetAPIType(name, pkg string) (i interface{}, t, permStruct, commonPermStruc
case "FullNode":
i = v0api.FullNodeStruct{}
t = reflect.TypeOf(new(struct{ v0api.FullNode })).Elem()
permStruct = reflect.TypeOf(v0api.FullNodeStruct{}.Internal)
commonPermStruct = reflect.TypeOf(v0api.CommonStruct{}.Internal)
permStruct = append(permStruct, reflect.TypeOf(v0api.FullNodeStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(v0api.CommonStruct{}.Internal))
permStruct = append(permStruct, reflect.TypeOf(v0api.NetStruct{}.Internal))
default:
panic("unknown type")
}

44
api/mocks/mock_full.go
View File

@ -105,6 +105,35 @@ func (mr *MockFullNodeMockRecorder) BeaconGetEntry(arg0, arg1 interface{}) *gomo
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BeaconGetEntry", reflect.TypeOf((*MockFullNode)(nil).BeaconGetEntry), arg0, arg1)
}
// ChainBlockstoreInfo mocks base method.
func (m *MockFullNode) ChainBlockstoreInfo(arg0 context.Context) (map[string]interface{}, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainBlockstoreInfo", arg0)
ret0, _ := ret[0].(map[string]interface{})
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChainBlockstoreInfo indicates an expected call of ChainBlockstoreInfo.
func (mr *MockFullNodeMockRecorder) ChainBlockstoreInfo(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainBlockstoreInfo", reflect.TypeOf((*MockFullNode)(nil).ChainBlockstoreInfo), arg0)
}
// ChainCheckBlockstore mocks base method.
func (m *MockFullNode) ChainCheckBlockstore(arg0 context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainCheckBlockstore", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// ChainCheckBlockstore indicates an expected call of ChainCheckBlockstore.
func (mr *MockFullNodeMockRecorder) ChainCheckBlockstore(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainCheckBlockstore", reflect.TypeOf((*MockFullNode)(nil).ChainCheckBlockstore), arg0)
}
// ChainDeleteObj mocks base method.
func (m *MockFullNode) ChainDeleteObj(arg0 context.Context, arg1 cid.Cid) error {
m.ctrl.T.Helper()
@ -194,6 +223,21 @@ func (mr *MockFullNodeMockRecorder) ChainGetMessage(arg0, arg1 interface{}) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetMessage", reflect.TypeOf((*MockFullNode)(nil).ChainGetMessage), arg0, arg1)
}
// ChainGetMessagesInTipset mocks base method.
func (m *MockFullNode) ChainGetMessagesInTipset(arg0 context.Context, arg1 types.TipSetKey) ([]api.Message, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainGetMessagesInTipset", arg0, arg1)
ret0, _ := ret[0].([]api.Message)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChainGetMessagesInTipset indicates an expected call of ChainGetMessagesInTipset.
func (mr *MockFullNodeMockRecorder) ChainGetMessagesInTipset(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetMessagesInTipset", reflect.TypeOf((*MockFullNode)(nil).ChainGetMessagesInTipset), arg0, arg1)
}
// ChainGetNode mocks base method.
func (m *MockFullNode) ChainGetNode(arg0 context.Context, arg1 string) (*api.IpldObject, error) {
m.ctrl.T.Helper()

17
api/permissioned.go
View File

@ -16,28 +16,33 @@ const (
var AllPermissions = []auth.Permission{PermRead, PermWrite, PermSign, PermAdmin}
var DefaultPerms = []auth.Permission{PermRead}
func permissionedProxies(in, out interface{}) {
outs := GetInternalStructs(out)
for _, o := range outs {
auth.PermissionedProxy(AllPermissions, DefaultPerms, in, o)
}
}
func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
var out StorageMinerStruct
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
permissionedProxies(a, &out)
return &out
}
func PermissionedFullAPI(a FullNode) FullNode {
var out FullNodeStruct
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.CommonStruct.Internal)
permissionedProxies(a, &out)
return &out
}
func PermissionedWorkerAPI(a Worker) Worker {
var out WorkerStruct
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
permissionedProxies(a, &out)
return &out
}
func PermissionedWalletAPI(a Wallet) Wallet {
var out WalletStruct
auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
permissionedProxies(a, &out)
return &out
}

2210
api/proxy_gen.go
View File

File diff suppressed because it is too large Load Diff

30
api/proxy_util.go Normal file
View File

@ -0,0 +1,30 @@
package api
import "reflect"
var _internalField = "Internal"
// GetInternalStructs extracts all pointers to 'Internal' sub-structs from the provided pointer to a proxy struct
func GetInternalStructs(in interface{}) []interface{} {
return getInternalStructs(reflect.ValueOf(in).Elem())
}
func getInternalStructs(rv reflect.Value) []interface{} {
var out []interface{}
internal := rv.FieldByName(_internalField)
ii := internal.Addr().Interface()
out = append(out, ii)
for i := 0; i < rv.NumField(); i++ {
if rv.Type().Field(i).Name == _internalField {
continue
}
sub := getInternalStructs(rv.Field(i))
out = append(out, sub...)
}
return out
}

62
api/proxy_util_test.go Normal file
View File

@ -0,0 +1,62 @@
package api
import (
"testing"
"github.com/stretchr/testify/require"
)
type StrA struct {
StrB
Internal struct {
A int
}
}
type StrB struct {
Internal struct {
B int
}
}
type StrC struct {
Internal struct {
Internal struct {
C int
}
}
}
func TestGetInternalStructs(t *testing.T) {
var proxy StrA
sts := GetInternalStructs(&proxy)
require.Len(t, sts, 2)
sa := sts[0].(*struct{ A int })
sa.A = 3
sb := sts[1].(*struct{ B int })
sb.B = 4
require.Equal(t, 3, proxy.Internal.A)
require.Equal(t, 4, proxy.StrB.Internal.B)
}
func TestNestedInternalStructs(t *testing.T) {
var proxy StrC
// check that only the top-level internal struct gets picked up
sts := GetInternalStructs(&proxy)
require.Len(t, sts, 1)
sa := sts[0].(*struct {
Internal struct {
C int
}
})
sa.Internal.C = 5
require.Equal(t, 5, proxy.Internal.Internal.C)
}

4
api/v0api/full.go
View File

@ -46,6 +46,7 @@ import (
// FullNode API is a low-level interface to the Filecoin network full node
type FullNode interface {
Common
Net
// MethodGroup: Chain
// The Chain method group contains methods for interacting with the
@ -92,6 +93,9 @@ type FullNode interface {
// specified block.
ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error) //perm:read
// ChainGetMessagesInTipset returns message stores in current tipset
ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]api.Message, error) //perm:read
// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
// If there are no blocks at the specified epoch, a tipset at an earlier epoch
// will be returned.

7
api/v0api/latest.go
View File

@ -5,8 +5,15 @@ import (
)
type Common = api.Common
type Net = api.Net
type CommonNet = api.CommonNet
type CommonStruct = api.CommonStruct
type CommonStub = api.CommonStub
type NetStruct = api.NetStruct
type NetStub = api.NetStub
type CommonNetStruct = api.CommonNetStruct
type CommonNetStub = api.CommonNetStub
type StorageMiner = api.StorageMiner
type StorageMinerStruct = api.StorageMinerStruct

1054
api/v0api/proxy_gen.go
View File

File diff suppressed because it is too large Load Diff

15
api/v0api/v0mocks/mock_full.go
View File

@ -194,6 +194,21 @@ func (mr *MockFullNodeMockRecorder) ChainGetMessage(arg0, arg1 interface{}) *gom
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetMessage", reflect.TypeOf((*MockFullNode)(nil).ChainGetMessage), arg0, arg1)
}
// ChainGetMessagesInTipset mocks base method.
func (m *MockFullNode) ChainGetMessagesInTipset(arg0 context.Context, arg1 types.TipSetKey) ([]api.Message, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainGetMessagesInTipset", arg0, arg1)
ret0, _ := ret[0].([]api.Message)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ChainGetMessagesInTipset indicates an expected call of ChainGetMessagesInTipset.
func (mr *MockFullNodeMockRecorder) ChainGetMessagesInTipset(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainGetMessagesInTipset", reflect.TypeOf((*MockFullNode)(nil).ChainGetMessagesInTipset), arg0, arg1)
}
// ChainGetNode mocks base method.
func (m *MockFullNode) ChainGetNode(arg0 context.Context, arg1 string) (*api.IpldObject, error) {
m.ctrl.T.Helper()

6
api/version.go
View File

@ -54,10 +54,10 @@ func VersionForType(nodeType NodeType) (Version, error) {
// semver versions of the rpc api exposed
var (
FullAPIVersion0 = newVer(1, 3, 0)
FullAPIVersion1 = newVer(2, 1, 0)
FullAPIVersion0 = newVer(1, 3, 1)
FullAPIVersion1 = newVer(2, 1, 1)
MinerAPIVersion0 = newVer(1, 1, 0)
MinerAPIVersion0 = newVer(1, 2, 0)
WorkerAPIVersion0 = newVer(1, 1, 0)
)

602
blockstore/badger/blockstore.go
View File

@ -4,11 +4,15 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"sync/atomic"
"sync"
"time"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"github.com/dgraph-io/badger/v2/pb"
"github.com/multiformats/go-base32"
"go.uber.org/zap"
@ -72,23 +76,45 @@ func (b *badgerLogger) Warningf(format string, args ...interface{}) {
b.skip2.Warnf(format, args...)
}
// bsState is the current blockstore state
type bsState int
const (
stateOpen int64 = iota
// stateOpen signifies an open blockstore
stateOpen bsState = iota
// stateClosing signifies a blockstore that is currently closing
stateClosing
// stateClosed signifies a blockstore that has been colosed
stateClosed
)
// Blockstore is a badger-backed IPLD blockstore.
//
// NOTE: once Close() is called, methods will try their best to return
// ErrBlockstoreClosed. This will guaranteed to happen for all subsequent
// operation calls after Close() has returned, but it may not happen for
// operations in progress. Those are likely to fail with a different error.
type Blockstore struct {
// state is accessed atomically
state int64
type bsMoveState int
DB *badger.DB
const (
// moveStateNone signifies that there is no move in progress
moveStateNone bsMoveState = iota
// moveStateMoving signifies that there is a move in a progress
moveStateMoving
// moveStateCleanup signifies that a move has completed or aborted and we are cleaning up
moveStateCleanup
// moveStateLock signifies that an exclusive lock has been acquired
moveStateLock
)
// Blockstore is a badger-backed IPLD blockstore.
type Blockstore struct {
stateLk sync.RWMutex
state bsState
viewers sync.WaitGroup
moveMx sync.Mutex
moveCond sync.Cond
moveState bsMoveState
rlock int
db *badger.DB
dbNext *badger.DB // when moving
opts Options
prefixing bool
prefix []byte
@ -97,6 +123,9 @@ type Blockstore struct {
var _ blockstore.Blockstore = (*Blockstore)(nil)
var _ blockstore.Viewer = (*Blockstore)(nil)
var _ blockstore.BlockstoreIterator = (*Blockstore)(nil)
var _ blockstore.BlockstoreGC = (*Blockstore)(nil)
var _ blockstore.BlockstoreSize = (*Blockstore)(nil)
var _ io.Closer = (*Blockstore)(nil)
// Open creates a new badger-backed blockstore, with the supplied options.
@ -111,73 +140,377 @@ func Open(opts Options) (*Blockstore, error) {
return nil, fmt.Errorf("failed to open badger blockstore: %w", err)
}
bs := &Blockstore{DB: db}
bs := &Blockstore{db: db, opts: opts}
if p := opts.Prefix; p != "" {
bs.prefixing = true
bs.prefix = []byte(p)
bs.prefixLen = len(bs.prefix)
}
bs.moveCond.L = &bs.moveMx
return bs, nil
}
// Close closes the store. If the store has already been closed, this noops and
// returns an error, even if the first closure resulted in error.
func (b *Blockstore) Close() error {
if !atomic.CompareAndSwapInt64(&b.state, stateOpen, stateClosing) {
b.stateLk.Lock()
if b.state != stateOpen {
b.stateLk.Unlock()
return nil
}
b.state = stateClosing
b.stateLk.Unlock()
defer atomic.StoreInt64(&b.state, stateClosed)
return b.DB.Close()
defer func() {
b.stateLk.Lock()
b.state = stateClosed
b.stateLk.Unlock()
}()
// wait for all accesses to complete
b.viewers.Wait()
return b.db.Close()
}
// CollectGarbage runs garbage collection on the value log
func (b *Blockstore) CollectGarbage() error {
if atomic.LoadInt64(&b.state) != stateOpen {
func (b *Blockstore) access() error {
b.stateLk.RLock()
defer b.stateLk.RUnlock()
if b.state != stateOpen {
return ErrBlockstoreClosed
}
var err error
b.viewers.Add(1)
return nil
}
func (b *Blockstore) isOpen() bool {
b.stateLk.RLock()
defer b.stateLk.RUnlock()
return b.state == stateOpen
}
// lockDB/unlockDB implement a recursive lock contingent on move state
func (b *Blockstore) lockDB() {
b.moveMx.Lock()
defer b.moveMx.Unlock()
if b.rlock == 0 {
for b.moveState == moveStateLock {
b.moveCond.Wait()
}
}
b.rlock++
}
func (b *Blockstore) unlockDB() {
b.moveMx.Lock()
defer b.moveMx.Unlock()
b.rlock--
if b.rlock == 0 && b.moveState == moveStateLock {
b.moveCond.Broadcast()
}
}
// lockMove/unlockMove implement an exclusive lock of move state
func (b *Blockstore) lockMove() {
b.moveMx.Lock()
b.moveState = moveStateLock
for b.rlock > 0 {
b.moveCond.Wait()
}
}
func (b *Blockstore) unlockMove(state bsMoveState) {
b.moveState = state
b.moveCond.Broadcast()
b.moveMx.Unlock()
}
// movingGC moves the blockstore to a new path, adjacent to the current path, and creates
// a symlink from the current path to the new path; the old blockstore is deleted.
//
// The blockstore MUST accept new writes during the move and ensure that these
// are persisted to the new blockstore; if a failure occurs aboring the move,
// then they must be peristed to the old blockstore.
// In short, the blockstore must not lose data from new writes during the move.
func (b *Blockstore) movingGC() error {
// this inlines moveLock/moveUnlock for the initial state check to prevent a second move
// while one is in progress without clobbering state
b.moveMx.Lock()
if b.moveState != moveStateNone {
b.moveMx.Unlock()
return fmt.Errorf("move in progress")
}
b.moveState = moveStateLock
for b.rlock > 0 {
b.moveCond.Wait()
}
b.moveState = moveStateMoving
b.moveCond.Broadcast()
b.moveMx.Unlock()
var path string
defer func() {
b.lockMove()
db2 := b.dbNext
b.dbNext = nil
var state bsMoveState
if db2 != nil {
state = moveStateCleanup
} else {
state = moveStateNone
}
b.unlockMove(state)
if db2 != nil {
err := db2.Close()
if err != nil {
log.Warnf("error closing badger db: %s", err)
}
b.deleteDB(path)
b.lockMove()
b.unlockMove(moveStateNone)
}
}()
// we resolve symlinks to create the new path in the adjacent to the old path.
// this allows the user to symlink the db directory into a separate filesystem.
basePath := b.opts.Dir
linkPath, err := filepath.EvalSymlinks(basePath)
if err != nil {
return fmt.Errorf("error resolving symlink %s: %w", basePath, err)
}
if basePath == linkPath {
path = basePath
} else {
name := filepath.Base(basePath)
dir := filepath.Dir(linkPath)
path = filepath.Join(dir, name)
}
path = fmt.Sprintf("%s.%d", path, time.Now().UnixNano())
log.Infof("moving blockstore from %s to %s", b.opts.Dir, path)
opts := b.opts
opts.Dir = path
opts.ValueDir = path
db2, err := badger.Open(opts.Options)
if err != nil {
return fmt.Errorf("failed to open badger blockstore in %s: %w", path, err)
}
b.lockMove()
b.dbNext = db2
b.unlockMove(moveStateMoving)
log.Info("copying blockstore")
err = b.doCopy(b.db, b.dbNext)
if err != nil {
return fmt.Errorf("error moving badger blockstore to %s: %w", path, err)
}
b.lockMove()
db1 := b.db
b.db = b.dbNext
b.dbNext = nil
b.unlockMove(moveStateCleanup)
err = db1.Close()
if err != nil {
log.Warnf("error closing old badger db: %s", err)
}
dbpath := b.opts.Dir
oldpath := fmt.Sprintf("%s.old.%d", dbpath, time.Now().Unix())
if err = os.Rename(dbpath, oldpath); err != nil {
// this is not catastrophic in the sense that we have not lost any data.
// but it is pretty bad, as the db path points to the old db, while we are now using to the new
// db; we can't continue and leave a ticking bomb for the next restart.
// so a panic is appropriate and user can fix.
panic(fmt.Errorf("error renaming old badger db dir from %s to %s: %w; USER ACTION REQUIRED", dbpath, oldpath, err)) //nolint
}
if err = os.Symlink(path, dbpath); err != nil {
// same here; the db path is pointing to the void. panic and let the user fix.
panic(fmt.Errorf("error symlinking new badger db dir from %s to %s: %w; USER ACTION REQUIRED", path, dbpath, err)) //nolint
}
b.deleteDB(oldpath)
log.Info("moving blockstore done")
return nil
}
// doCopy copies a badger blockstore to another, with an optional filter; if the filter
// is not nil, then only cids that satisfy the filter will be copied.
func (b *Blockstore) doCopy(from, to *badger.DB) error {
workers := runtime.NumCPU() / 2
if workers < 2 {
workers = 2
}
stream := from.NewStream()
stream.NumGo = workers
stream.LogPrefix = "doCopy"
stream.Send = func(list *pb.KVList) error {
batch := to.NewWriteBatch()
defer batch.Cancel()
for _, kv := range list.Kv {
if kv.Key == nil || kv.Value == nil {
continue
}
if err := batch.Set(kv.Key, kv.Value); err != nil {
return err
}
}
return batch.Flush()
}
return stream.Orchestrate(context.Background())
}
func (b *Blockstore) deleteDB(path string) {
// follow symbolic links, otherwise the data wil be left behind
lpath, err := filepath.EvalSymlinks(path)
if err != nil {
log.Warnf("error resolving symlinks in %s", path)
return
}
log.Infof("removing data directory %s", lpath)
if err := os.RemoveAll(lpath); err != nil {
log.Warnf("error deleting db at %s: %s", lpath, err)
return
}
if path != lpath {
log.Infof("removing link %s", path)
if err := os.Remove(path); err != nil {
log.Warnf("error removing symbolic link %s", err)
}
}
}
func (b *Blockstore) onlineGC() error {
b.lockDB()
defer b.unlockDB()
// compact first to gather the necessary statistics for GC
nworkers := runtime.NumCPU() / 2
if nworkers < 2 {
nworkers = 2
}
err := b.db.Flatten(nworkers)
if err != nil {
return err
}
for err == nil {
err = b.DB.RunValueLogGC(0.125)
err = b.db.RunValueLogGC(0.125)
}
if err == badger.ErrNoRewrite {
// not really an error in this case
// not really an error in this case, it signals the end of GC
return nil
}
return err
}
// Compact runs a synchronous compaction
func (b *Blockstore) Compact() error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
// CollectGarbage compacts and runs garbage collection on the value log;
// implements the BlockstoreGC trait
func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
var options blockstore.BlockstoreGCOptions
for _, opt := range opts {
err := opt(&options)
if err != nil {
return err
}
}
nworkers := runtime.NumCPU() / 2
if nworkers < 2 {
nworkers = 2
if options.FullGC {
return b.movingGC()
}
return b.DB.Flatten(nworkers)
return b.onlineGC()
}
// Size returns the aggregate size of the blockstore
func (b *Blockstore) Size() (int64, error) {
if err := b.access(); err != nil {
return 0, err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
lsm, vlog := b.db.Size()
size := lsm + vlog
if size == 0 {
// badger reports a 0 size on symlinked directories... sigh
dir := b.opts.Dir
entries, err := os.ReadDir(dir)
if err != nil {
return 0, err
}
for _, e := range entries {
path := filepath.Join(dir, e.Name())
finfo, err := os.Stat(path)
if err != nil {
return 0, err
}
size += finfo.Size()
}
}
return size, nil
}
// View implements blockstore.Viewer, which leverages zero-copy read-only
// access to values.
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(cid)
if pooled {
defer KeyPool.Put(k)
}
return b.DB.View(func(txn *badger.Txn) error {
return b.db.View(func(txn *badger.Txn) error {
switch item, err := txn.Get(k); err {
case nil:
return item.Value(fn)
@ -191,16 +524,20 @@ func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
// Has implements Blockstore.Has.
func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
if atomic.LoadInt64(&b.state) != stateOpen {
return false, ErrBlockstoreClosed
if err := b.access(); err != nil {
return false, err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(cid)
if pooled {
defer KeyPool.Put(k)
}
err := b.DB.View(func(txn *badger.Txn) error {
err := b.db.View(func(txn *badger.Txn) error {
_, err := txn.Get(k)
return err
})
@ -221,9 +558,13 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
return nil, blockstore.ErrNotFound
}
if atomic.LoadInt64(&b.state) != stateOpen {
return nil, ErrBlockstoreClosed
if err := b.access(); err != nil {
return nil, err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(cid)
if pooled {
@ -231,7 +572,7 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
}
var val []byte
err := b.DB.View(func(txn *badger.Txn) error {
err := b.db.View(func(txn *badger.Txn) error {
switch item, err := txn.Get(k); err {
case nil:
val, err = item.ValueCopy(nil)
@ -250,9 +591,13 @@ func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
// GetSize implements Blockstore.GetSize.
func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
if atomic.LoadInt64(&b.state) != stateOpen {
return -1, ErrBlockstoreClosed
if err := b.access(); err != nil {
return 0, err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(cid)
if pooled {
@ -260,7 +605,7 @@ func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
}
var size int
err := b.DB.View(func(txn *badger.Txn) error {
err := b.db.View(func(txn *badger.Txn) error {
switch item, err := txn.Get(k); err {
case nil:
size = int(item.ValueSize())
@ -279,29 +624,52 @@ func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
// Put implements Blockstore.Put.
func (b *Blockstore) Put(block blocks.Block) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(block.Cid())
if pooled {
defer KeyPool.Put(k)
}
err := b.DB.Update(func(txn *badger.Txn) error {
return txn.Set(k, block.RawData())
})
if err != nil {
err = fmt.Errorf("failed to put block in badger blockstore: %w", err)
put := func(db *badger.DB) error {
err := db.Update(func(txn *badger.Txn) error {
return txn.Set(k, block.RawData())
})
if err != nil {
return fmt.Errorf("failed to put block in badger blockstore: %w", err)
}
return nil
}
return err
if err := put(b.db); err != nil {
return err
}
if b.dbNext != nil {
if err := put(b.dbNext); err != nil {
return err
}
}
return nil
}
// PutMany implements Blockstore.PutMany.
func (b *Blockstore) PutMany(blocks []blocks.Block) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
// toReturn tracks the byte slices to return to the pool, if we're using key
// prefixing. we can't return each slice to the pool after each Set, because
@ -316,46 +684,75 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
}()
}
batch := b.DB.NewWriteBatch()
defer batch.Cancel()
keys := make([][]byte, 0, len(blocks))
for _, block := range blocks {
k, pooled := b.PooledStorageKey(block.Cid())
if pooled {
toReturn = append(toReturn, k)
}
if err := batch.Set(k, block.RawData()); err != nil {
keys = append(keys, k)
}
put := func(db *badger.DB) error {
batch := db.NewWriteBatch()
defer batch.Cancel()
for i, block := range blocks {
k := keys[i]
if err := batch.Set(k, block.RawData()); err != nil {
return err
}
}
err := batch.Flush()
if err != nil {
return fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
}
return nil
}
if err := put(b.db); err != nil {
return err
}
if b.dbNext != nil {
if err := put(b.dbNext); err != nil {
return err
}
}
err := batch.Flush()
if err != nil {
err = fmt.Errorf("failed to put blocks in badger blockstore: %w", err)
}
return err
return nil
}
// DeleteBlock implements Blockstore.DeleteBlock.
func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
k, pooled := b.PooledStorageKey(cid)
if pooled {
defer KeyPool.Put(k)
}
return b.DB.Update(func(txn *badger.Txn) error {
return b.db.Update(func(txn *badger.Txn) error {
return txn.Delete(k)
})
}
func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
if atomic.LoadInt64(&b.state) != stateOpen {
return ErrBlockstoreClosed
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
// toReturn tracks the byte slices to return to the pool, if we're using key
// prefixing. we can't return each slice to the pool after each Set, because
@ -370,7 +767,7 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
}()
}
batch := b.DB.NewWriteBatch()
batch := b.db.NewWriteBatch()
defer batch.Cancel()
for _, cid := range cids {
@ -392,11 +789,14 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
// AllKeysChan implements Blockstore.AllKeysChan.
func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
if atomic.LoadInt64(&b.state) != stateOpen {
return nil, ErrBlockstoreClosed
if err := b.access(); err != nil {
return nil, err
}
txn := b.DB.NewTransaction(false)
b.lockDB()
defer b.unlockDB()
txn := b.db.NewTransaction(false)
opts := badger.IteratorOptions{PrefetchSize: 100}
if b.prefixing {
opts.Prefix = b.prefix
@ -405,6 +805,7 @@ func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
ch := make(chan cid.Cid)
go func() {
defer b.viewers.Done()
defer close(ch)
defer iter.Close()
@ -415,7 +816,7 @@ func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
if ctx.Err() != nil {
return // context has fired.
}
if atomic.LoadInt64(&b.state) != stateOpen {
if !b.isOpen() {
// open iterators will run even after the database is closed...
return // closing, yield.
}
@ -442,6 +843,59 @@ func (b *Blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return ch, nil
}
// Implementation of BlockstoreIterator interface
func (b *Blockstore) ForEachKey(f func(cid.Cid) error) error {
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
txn := b.db.NewTransaction(false)
defer txn.Discard()
opts := badger.IteratorOptions{PrefetchSize: 100}
if b.prefixing {
opts.Prefix = b.prefix
}
iter := txn.NewIterator(opts)
defer iter.Close()
var buf []byte
for iter.Rewind(); iter.Valid(); iter.Next() {
if !b.isOpen() {
return ErrBlockstoreClosed
}
k := iter.Item().Key()
if b.prefixing {
k = k[b.prefixLen:]
}
klen := base32.RawStdEncoding.DecodedLen(len(k))
if klen > len(buf) {
buf = make([]byte, klen)
}
n, err := base32.RawStdEncoding.Decode(buf, k)
if err != nil {
return err
}
c := cid.NewCidV1(cid.Raw, buf[:n])
err = f(c)
if err != nil {
return err
}
}
return nil
}
// HashOnRead implements Blockstore.HashOnRead. It is not supported by this
// blockstore.
func (b *Blockstore) HashOnRead(_ bool) {
@ -494,3 +948,9 @@ func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte {
}
return dst[:reqsize]
}
// this method is added for lotus-shed needs
// WARNING: THIS IS COMPLETELY UNSAFE; DONT USE THIS IN PRODUCTION CODE
func (b *Blockstore) DB() *badger.DB {
return b.db
}

171
blockstore/badger/blockstore_test.go
View File

@ -1,12 +1,19 @@
package badgerbs
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
blocks "github.com/ipfs/go-block-format"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/blockstore"
)
@ -89,3 +96,165 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
return Open(optsSupplier(path))
}
}
func testMove(t *testing.T, optsF func(string) Options) {
basePath, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
dbPath := filepath.Join(basePath, "db")
t.Cleanup(func() {
_ = os.RemoveAll(basePath)
})
db, err := Open(optsF(dbPath))
if err != nil {
t.Fatal(err)
}
defer db.Close() //nolint
var have []blocks.Block
var deleted []cid.Cid
// add some blocks
for i := 0; i < 10; i++ {
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := db.Put(blk)
if err != nil {
t.Fatal(err)
}
have = append(have, blk)
}
// delete some of them
for i := 5; i < 10; i++ {
c := have[i].Cid()
err := db.DeleteBlock(c)
if err != nil {
t.Fatal(err)
}
deleted = append(deleted, c)
}
have = have[:5]
// start a move concurrent with some more puts
g := new(errgroup.Group)
g.Go(func() error {
for i := 10; i < 1000; i++ {
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := db.Put(blk)
if err != nil {
return err
}
have = append(have, blk)
}
return nil
})
g.Go(func() error {
return db.CollectGarbage(blockstore.WithFullGC(true))
})
err = g.Wait()
if err != nil {
t.Fatal(err)
}
// now check that we have all the blocks in have and none in the deleted lists
checkBlocks := func() {
for _, blk := range have {
has, err := db.Has(blk.Cid())
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("missing block")
}
blk2, err := db.Get(blk.Cid())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(blk.RawData(), blk2.RawData()) {
t.Fatal("data mismatch")
}
}
for _, c := range deleted {
has, err := db.Has(c)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("resurrected block")
}
}
}
checkBlocks()
// check the basePath -- it should contain a directory with name db.{timestamp}, soft-linked
// to db and nothing else
checkPath := func() {
entries, err := os.ReadDir(basePath)
if err != nil {
t.Fatal(err)
}
if len(entries) != 2 {
t.Fatalf("too many entries; expected %d but got %d", 2, len(entries))
}
var haveDB, haveDBLink bool
for _, e := range entries {
if e.Name() == "db" {
if (e.Type() & os.ModeSymlink) == 0 {
t.Fatal("found db, but it's not a symlink")
}
haveDBLink = true
continue
}
if strings.HasPrefix(e.Name(), "db.") {
if !e.Type().IsDir() {
t.Fatal("found db prefix, but it's not a directory")
}
haveDB = true
continue
}
}
if !haveDB {
t.Fatal("db directory is missing")
}
if !haveDBLink {
t.Fatal("db link is missing")
}
}
checkPath()
// now do another FullGC to test the double move and following of symlinks
if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
t.Fatal(err)
}
checkBlocks()
checkPath()
}
func TestMoveNoPrefix(t *testing.T) {
testMove(t, DefaultOptions)
}
func TestMoveWithPrefix(t *testing.T) {
testMove(t, func(path string) Options {
opts := DefaultOptions(path)
opts.Prefix = "/prefixed/"
return opts
})
}

30
blockstore/blockstore.go
View File

@ -30,6 +30,36 @@ type BatchDeleter interface {
DeleteMany(cids []cid.Cid) error
}
// BlockstoreIterator is a trait for efficient iteration
type BlockstoreIterator interface {
ForEachKey(func(cid.Cid) error) error
}
// BlockstoreGC is a trait for blockstores that support online garbage collection
type BlockstoreGC interface {
CollectGarbage(options ...BlockstoreGCOption) error
}
// BlockstoreGCOption is a functional interface for controlling blockstore GC options
type BlockstoreGCOption = func(*BlockstoreGCOptions) error
// BlockstoreGCOptions is a struct with GC options
type BlockstoreGCOptions struct {
FullGC bool
}
func WithFullGC(fullgc bool) BlockstoreGCOption {
return func(opts *BlockstoreGCOptions) error {
opts.FullGC = fullgc
return nil
}
}
// BlockstoreSize is a trait for on-disk blockstores that can report their size
type BlockstoreSize interface {
Size() (int64, error)
}
// WrapIDStore wraps the underlying blockstore in an "identity" blockstore.
// The ID store filters out all puts for blocks with CIDs using the "identity"
// hash function. It also extracts inlined blocks from CIDs using the identity

66
blockstore/discard.go Normal file
View File

@ -0,0 +1,66 @@
package blockstore
import (
"context"
"io"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
var _ Blockstore = (*discardstore)(nil)
type discardstore struct {
bs Blockstore
}
func NewDiscardStore(bs Blockstore) Blockstore {
return &discardstore{bs: bs}
}
func (b *discardstore) Has(cid cid.Cid) (bool, error) {
return b.bs.Has(cid)
}
func (b *discardstore) HashOnRead(hor bool) {
b.bs.HashOnRead(hor)
}
func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
return b.bs.Get(cid)
}
func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
return b.bs.GetSize(cid)
}
func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
return b.bs.View(cid, f)
}
func (b *discardstore) Put(blk blocks.Block) error {
return nil
}
func (b *discardstore) PutMany(blks []blocks.Block) error {
return nil
}
func (b *discardstore) DeleteBlock(cid cid.Cid) error {
return nil
}
func (b *discardstore) DeleteMany(cids []cid.Cid) error {
return nil
}
func (b *discardstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return b.bs.AllKeysChan(ctx)
}
func (b *discardstore) Close() error {
if c, ok := b.bs.(io.Closer); ok {
return c.Close()
}
return nil
}

124
blockstore/splitstore/README.md Normal file
View File

@ -0,0 +1,124 @@
# SplitStore: An actively scalable blockstore for the Filecoin chain
The SplitStore was first introduced in lotus v1.5.1, as an experiment
in reducing the performance impact of large blockstores.
With lotus v1.11.1, we introduce the next iteration in design and
implementation, which we call SplitStore v1.
The new design (see [#6474](https://github.com/filecoin-project/lotus/pull/6474)
evolves the splitstore to be a freestanding compacting blockstore that
allows us to keep a small (60-100GB) working set in a hot blockstore
and reliably archive out of scope objects in a coldstore. The
coldstore can also be a discard store, whereby out of scope objects
are discarded or a regular badger blockstore (the default), which can
be periodically garbage collected according to configurable user
retention policies.
To enable the splitstore, edit `.lotus/config.toml` and add the following:
```
[Chainstore]
EnableSplitstore = true
```
If you intend to use the discard coldstore, your also need to add the following:
```
[Chainstore.Splitstore]
ColdStoreType = "discard"
```
In general you _should not_ have to use the discard store, unless you
are running a network assistive node (like a bootstrapper or booster)
or have very constrained hardware with not enough disk space to
maintain a coldstore, even with garbage collection. It is also appropriate
for small nodes that are simply watching the chain.
*Warning:* Using the discard store for a general purpose node is discouraged, unless
you really know what you are doing. Use it at your own risk.
## Configuration Options
These are options in the `[Chainstore.Splitstore]` section of the configuration:
- `HotStoreType` -- specifies the type of hotstore to use.
The only currently supported option is `"badger"`.
- `ColdStoreType` -- specifies the type of coldstore to use.
The default value is `"universal"`, which will use the initial monolith blockstore
as the coldstore.
The other possible value is `"discard"`, as outlined above, which is specialized for
running without a coldstore. Note that the discard store wraps the initial monolith
blockstore and discards writes; this is necessary to support syncing from a snapshot.
- `MarkSetType` -- specifies the type of markset to use during compaction.
The markset is the data structure used by compaction/gc to track live objects.
The default value is `"map"`, which will use an in-memory map; if you are limited
in memory (or indeed see compaction run out of memory), you can also specify
`"badger"` which will use an disk backed markset, using badger. This will use
much less memory, but will also make compaction slower.
- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
finalities maintained by default, to maintain messages and message receipts in the
hotstore. This is useful for assistive nodes that want to support syncing for other
nodes beyond 4 finalities, while running with the discard coldstore option.
It is also useful for miners who accept deals and need to lookback messages beyond
the 4 finalities, which would otherwise hit the coldstore.
- `HotStoreFullGCFrequency` -- specifies how frequenty to garbage collect the hotstore
using full (moving) GC.
The default value is 20, which uses full GC every 20 compactions (about once a week);
set to 0 to disable full GC altogether.
Rationale: badger supports online GC, and this is used by default. However it has proven to
be ineffective in practice with the hotstore size slowly creeping up. In order to address this,
we have added moving GC support in our badger wrapper, which can effectively reclaim all space.
The downside is that it takes a bit longer to perform a moving GC and you also need enough
space to house the new hotstore while the old one is still live.
## Operation
When the splitstore is first enabled, the existing blockstore becomes
the coldstore and a fresh hotstore is initialized.
The hotstore is warmed up on first startup so as to load all chain
headers and state roots in the current head. This allows us to
immediately gain the performance benefits of a smallerblockstore which
can be substantial for full archival nodes.
All new writes are directed to the hotstore, while reads first hit the
hotstore, with fallback to the coldstore.
Once 5 finalities have ellapsed, and every finality henceforth, the
blockstore _compacts_. Compaction is the process of moving all
unreachable objects within the last 4 finalities from the hotstore to
the coldstore. If the system is configured with a discard coldstore,
these objects are discarded. Note that chain headers, all the way to
genesis, are considered reachable. Stateroots and messages are
considered reachable only within the last 4 finalities, unless there
is a live reference to them.
## Compaction
Compaction works transactionally with the following algorithm:
- We prepare a transaction, whereby all i/o referenced objects through the API are tracked.
- We walk the chain and mark reachable objects, keeping 4 finalities of state roots and messages and all headers all the way to genesis.
- Once the chain walk is complete, we begin full transaction protection with concurrent marking; we walk and mark all references created during the chain walk. On the same time, all I/O through the API concurrently marks objects as live references.
- We collect cold objects by iterating through the hotstore and checking the mark set; if an object is not marked, then it is candidate for purge.
- When running with a coldstore, we next copy all cold objects to the coldstore.
- At this point we are ready to begin purging:
- We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
- We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
- We then end the transaction and compact/gc the hotstore.
## Garbage Collection
TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
## Utilities
`lotus-shed` has a `splitstore` command which provides some utilities:
- `rollback` -- rolls back a splitstore installation.
This command copies the hotstore on top of the coldstore, and then deletes the splitstore
directory and associated metadata keys.
It can also optionally compact/gc the coldstore after the copy (with the `--gc-coldstore` flag)
and automatically rewrite the lotus config to disable splitstore (with the `--rewrite-config` flag).
Note: the node *must be stopped* before running this command.
- `check` -- asynchronously runs a basic healthcheck on the splitstore.
The results are appended to `<lotus-repo>/datastore/splitstore/check.txt`.
- `info` -- prints some basic information about the splitstore.

273
blockstore/splitstore/debug.go Normal file
View File

@ -0,0 +1,273 @@
package splitstore
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime/debug"
"strings"
"sync"
"time"
"go.uber.org/multierr"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
type debugLog struct {
readLog, writeLog, deleteLog, stackLog *debugLogOp
stackMx sync.Mutex
stackMap map[string]string
}
type debugLogOp struct {
path string
mx sync.Mutex
log *os.File
count int
}
func openDebugLog(path string) (*debugLog, error) {
basePath := filepath.Join(path, "debug")
err := os.MkdirAll(basePath, 0755)
if err != nil {
return nil, err
}
readLog, err := openDebugLogOp(basePath, "read.log")
if err != nil {
return nil, err
}
writeLog, err := openDebugLogOp(basePath, "write.log")
if err != nil {
_ = readLog.Close()
return nil, err
}
deleteLog, err := openDebugLogOp(basePath, "delete.log")
if err != nil {
_ = readLog.Close()
_ = writeLog.Close()
return nil, err
}
stackLog, err := openDebugLogOp(basePath, "stack.log")
if err != nil {
_ = readLog.Close()
_ = writeLog.Close()
_ = deleteLog.Close()
return nil, xerrors.Errorf("error opening stack log: %w", err)
}
return &debugLog{
readLog: readLog,
writeLog: writeLog,
deleteLog: deleteLog,
stackLog: stackLog,
stackMap: make(map[string]string),
}, nil
}
func (d *debugLog) LogReadMiss(cid cid.Cid) {
if d == nil {
return
}
stack := d.getStack()
err := d.readLog.Log("%s %s %s\n", d.timestamp(), cid, stack)
if err != nil {
log.Warnf("error writing read log: %s", err)
}
}
func (d *debugLog) LogWrite(blk blocks.Block) {
if d == nil {
return
}
var stack string
if enableDebugLogWriteTraces {
stack = " " + d.getStack()
}
err := d.writeLog.Log("%s %s%s\n", d.timestamp(), blk.Cid(), stack)
if err != nil {
log.Warnf("error writing write log: %s", err)
}
}
func (d *debugLog) LogWriteMany(blks []blocks.Block) {
if d == nil {
return
}
var stack string
if enableDebugLogWriteTraces {
stack = " " + d.getStack()
}
now := d.timestamp()
for _, blk := range blks {
err := d.writeLog.Log("%s %s%s\n", now, blk.Cid(), stack)
if err != nil {
log.Warnf("error writing write log: %s", err)
break
}
}
}
func (d *debugLog) LogDelete(cids []cid.Cid) {
if d == nil {
return
}
now := d.timestamp()
for _, c := range cids {
err := d.deleteLog.Log("%s %s\n", now, c)
if err != nil {
log.Warnf("error writing delete log: %s", err)
break
}
}
}
func (d *debugLog) Flush() {
if d == nil {
return
}
// rotate non-empty logs
d.readLog.Rotate()
d.writeLog.Rotate()
d.deleteLog.Rotate()
d.stackLog.Rotate()
}
func (d *debugLog) Close() error {
if d == nil {
return nil
}
err1 := d.readLog.Close()
err2 := d.writeLog.Close()
err3 := d.deleteLog.Close()
err4 := d.stackLog.Close()
return multierr.Combine(err1, err2, err3, err4)
}
func (d *debugLog) getStack() string {
sk := d.getNormalizedStackTrace()
hash := sha256.Sum256([]byte(sk))
key := string(hash[:])
d.stackMx.Lock()
repr, ok := d.stackMap[key]
if !ok {
repr = hex.EncodeToString(hash[:])
d.stackMap[key] = repr
err := d.stackLog.Log("%s\n%s\n", repr, sk)
if err != nil {
log.Warnf("error writing stack trace for %s: %s", repr, err)
}
}
d.stackMx.Unlock()
return repr
}
func (d *debugLog) getNormalizedStackTrace() string {
sk := string(debug.Stack())
// Normalization for deduplication
// skip first line -- it's the goroutine
// for each line that ends in a ), remove the call args -- these are the registers
lines := strings.Split(sk, "\n")[1:]
for i, line := range lines {
if len(line) > 0 && line[len(line)-1] == ')' {
idx := strings.LastIndex(line, "(")
if idx < 0 {
continue
}
lines[i] = line[:idx]
}
}
return strings.Join(lines, "\n")
}
func (d *debugLog) timestamp() string {
ts, _ := time.Now().MarshalText()
return string(ts)
}
func openDebugLogOp(basePath, name string) (*debugLogOp, error) {
path := filepath.Join(basePath, name)
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return nil, xerrors.Errorf("error opening %s: %w", name, err)
}
return &debugLogOp{path: path, log: file}, nil
}
func (d *debugLogOp) Close() error {
d.mx.Lock()
defer d.mx.Unlock()
return d.log.Close()
}
func (d *debugLogOp) Log(template string, arg ...interface{}) error {
d.mx.Lock()
defer d.mx.Unlock()
d.count++
_, err := fmt.Fprintf(d.log, template, arg...)
return err
}
func (d *debugLogOp) Rotate() {
d.mx.Lock()
defer d.mx.Unlock()
if d.count == 0 {
return
}
err := d.log.Close()
if err != nil {
log.Warnf("error closing log (file: %s): %s", d.path, err)
return
}
arxivPath := fmt.Sprintf("%s-%d", d.path, time.Now().Unix())
err = os.Rename(d.path, arxivPath)
if err != nil {
log.Warnf("error moving log (file: %s): %s", d.path, err)
return
}
go func() {
cmd := exec.Command("gzip", arxivPath)
err := cmd.Run()
if err != nil {
log.Warnf("error compressing log (file: %s): %s", arxivPath, err)
}
}()
d.count = 0
d.log, err = os.OpenFile(d.path, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
log.Warnf("error opening log (file: %s): %s", d.path, err)
return
}
}

18
blockstore/splitstore/markset.go
View File

@ -1,26 +1,26 @@
package splitstore
import (
"path/filepath"
"errors"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
var errMarkSetClosed = errors.New("markset closed")
// MarkSet is a utility to keep track of seen CID, and later query for them.
//
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
// * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).
// * If a probabilistic result is acceptable, it can be backed by a bloom filter
type MarkSet interface {
Mark(cid.Cid) error
Has(cid.Cid) (bool, error)
Close() error
SetConcurrent()
}
// markBytes is deliberately a non-nil empty byte slice for serialization.
var markBytes = []byte{}
type MarkSetEnv interface {
Create(name string, sizeHint int64) (MarkSet, error)
Close() error
@ -28,10 +28,12 @@ type MarkSetEnv interface {
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
switch mtype {
case "", "bloom":
case "bloom":
return NewBloomMarkSetEnv()
case "bolt":
return NewBoltMarkSetEnv(filepath.Join(path, "markset.bolt"))
case "map":
return NewMapMarkSetEnv()
case "badger":
return NewBadgerMarkSetEnv(path)
default:
return nil, xerrors.Errorf("unknown mark set type %s", mtype)
}

230
blockstore/splitstore/markset_badger.go Normal file
View File

@ -0,0 +1,230 @@
package splitstore
import (
"os"
"path/filepath"
"sync"
"golang.org/x/xerrors"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"go.uber.org/zap"
cid "github.com/ipfs/go-cid"
)
type BadgerMarkSetEnv struct {
path string
}
var _ MarkSetEnv = (*BadgerMarkSetEnv)(nil)
type BadgerMarkSet struct {
mx sync.RWMutex
cond sync.Cond
pend map[string]struct{}
writing map[int]map[string]struct{}
writers int
seqno int
db *badger.DB
path string
}
var _ MarkSet = (*BadgerMarkSet)(nil)
var badgerMarkSetBatchSize = 16384
func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
msPath := filepath.Join(path, "markset.badger")
err := os.MkdirAll(msPath, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
return &BadgerMarkSetEnv{path: msPath}, nil
}
func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
path := filepath.Join(e.path, name)
// clean up first
err := os.RemoveAll(path)
if err != nil {
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
}
err = os.MkdirAll(path, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
opts := badger.DefaultOptions(path)
opts.SyncWrites = false
opts.CompactL0OnClose = false
opts.Compression = options.None
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
// between the system blockstore and the markset.
// It was observed that using the default memory mapped option resulted in
// significant interference and unacceptably high block validation times once the markset
// exceeded 1GB in size.
opts.TableLoadingMode = options.FileIO
opts.ValueLogLoadingMode = options.FileIO
opts.Logger = &badgerLogger{
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
}
db, err := badger.Open(opts)
if err != nil {
return nil, xerrors.Errorf("error creating badger markset: %w", err)
}
ms := &BadgerMarkSet{
pend: make(map[string]struct{}),
writing: make(map[int]map[string]struct{}),
db: db,
path: path,
}
ms.cond.L = &ms.mx
return ms, nil
}
func (e *BadgerMarkSetEnv) Close() error {
return os.RemoveAll(e.path)
}
func (s *BadgerMarkSet) Mark(c cid.Cid) error {
s.mx.Lock()
if s.pend == nil {
s.mx.Unlock()
return errMarkSetClosed
}
s.pend[string(c.Hash())] = struct{}{}
if len(s.pend) < badgerMarkSetBatchSize {
s.mx.Unlock()
return nil
}
pend := s.pend
seqno := s.seqno
s.seqno++
s.writing[seqno] = pend
s.pend = make(map[string]struct{})
s.writers++
s.mx.Unlock()
defer func() {
s.mx.Lock()
defer s.mx.Unlock()
delete(s.writing, seqno)
s.writers--
if s.writers == 0 {
s.cond.Broadcast()
}
}()
empty := []byte{} // not nil
batch := s.db.NewWriteBatch()
defer batch.Cancel()
for k := range pend {
if err := batch.Set([]byte(k), empty); err != nil {
return err
}
}
err := batch.Flush()
if err != nil {
return xerrors.Errorf("error flushing batch to badger markset: %w", err)
}
return nil
}
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
s.mx.RLock()
defer s.mx.RUnlock()
if s.pend == nil {
return false, errMarkSetClosed
}
key := c.Hash()
pendKey := string(key)
_, ok := s.pend[pendKey]
if ok {
return true, nil
}
for _, wr := range s.writing {
_, ok := wr[pendKey]
if ok {
return true, nil
}
}
err := s.db.View(func(txn *badger.Txn) error {
_, err := txn.Get(key)
return err
})
switch err {
case nil:
return true, nil
case badger.ErrKeyNotFound:
return false, nil
default:
return false, xerrors.Errorf("error checking badger markset: %w", err)
}
}
func (s *BadgerMarkSet) Close() error {
s.mx.Lock()
defer s.mx.Unlock()
if s.pend == nil {
return nil
}
for s.writers > 0 {
s.cond.Wait()
}
s.pend = nil
db := s.db
s.db = nil
err := db.Close()
if err != nil {
return xerrors.Errorf("error closing badger markset: %w", err)
}
err = os.RemoveAll(s.path)
if err != nil {
return xerrors.Errorf("error deleting badger markset: %w", err)
}
return nil
}
func (s *BadgerMarkSet) SetConcurrent() {}
// badger logging through go-log
type badgerLogger struct {
*zap.SugaredLogger
skip2 *zap.SugaredLogger
}
func (b *badgerLogger) Warningf(format string, args ...interface{}) {}
func (b *badgerLogger) Infof(format string, args ...interface{}) {}
func (b *badgerLogger) Debugf(format string, args ...interface{}) {}

30
blockstore/splitstore/markset_bloom.go
View File

@ -3,6 +3,7 @@ package splitstore
import (
"crypto/rand"
"crypto/sha256"
"sync"
"golang.org/x/xerrors"
@ -21,7 +22,9 @@ var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
type BloomMarkSet struct {
salt []byte
mx sync.RWMutex
bf *bbloom.Bloom
ts bool
}
var _ MarkSet = (*BloomMarkSet)(nil)
@ -64,14 +67,41 @@ func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
}
func (s *BloomMarkSet) Mark(cid cid.Cid) error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.bf == nil {
return errMarkSetClosed
}
s.bf.Add(s.saltedKey(cid))
return nil
}
func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
if s.ts {
s.mx.RLock()
defer s.mx.RUnlock()
}
if s.bf == nil {
return false, errMarkSetClosed
}
return s.bf.Has(s.saltedKey(cid)), nil
}
func (s *BloomMarkSet) Close() error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
s.bf = nil
return nil
}
func (s *BloomMarkSet) SetConcurrent() {
s.ts = true
}

81
blockstore/splitstore/markset_bolt.go
View File

@ -1,81 +0,0 @@
package splitstore
import (
"time"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bolt "go.etcd.io/bbolt"
)
type BoltMarkSetEnv struct {
db *bolt.DB
}
var _ MarkSetEnv = (*BoltMarkSetEnv)(nil)
type BoltMarkSet struct {
db *bolt.DB
bucketId []byte
}
var _ MarkSet = (*BoltMarkSet)(nil)
func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error) {
db, err := bolt.Open(path, 0644,
&bolt.Options{
Timeout: 1 * time.Second,
NoSync: true,
})
if err != nil {
return nil, err
}
return &BoltMarkSetEnv{db: db}, nil
}
func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error) {
bucketId := []byte(name)
err := e.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucketId)
if err != nil {
return xerrors.Errorf("error creating bolt db bucket %s: %w", name, err)
}
return nil
})
if err != nil {
return nil, err
}
return &BoltMarkSet{db: e.db, bucketId: bucketId}, nil
}
func (e *BoltMarkSetEnv) Close() error {
return e.db.Close()
}
func (s *BoltMarkSet) Mark(cid cid.Cid) error {
return s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Put(cid.Hash(), markBytes)
})
}
func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
v := b.Get(cid.Hash())
result = v != nil
return nil
})
return result, err
}
func (s *BoltMarkSet) Close() error {
return s.db.Update(func(tx *bolt.Tx) error {
return tx.DeleteBucket(s.bucketId)
})
}

75
blockstore/splitstore/markset_map.go Normal file
View File

@ -0,0 +1,75 @@
package splitstore
import (
"sync"
cid "github.com/ipfs/go-cid"
)
type MapMarkSetEnv struct{}
var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
type MapMarkSet struct {
mx sync.RWMutex
set map[string]struct{}
ts bool
}
var _ MarkSet = (*MapMarkSet)(nil)
func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
return &MapMarkSetEnv{}, nil
}
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
return &MapMarkSet{
set: make(map[string]struct{}, sizeHint),
}, nil
}
func (e *MapMarkSetEnv) Close() error {
return nil
}
func (s *MapMarkSet) Mark(cid cid.Cid) error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.set == nil {
return errMarkSetClosed
}
s.set[string(cid.Hash())] = struct{}{}
return nil
}
func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
if s.ts {
s.mx.RLock()
defer s.mx.RUnlock()
}
if s.set == nil {
return false, errMarkSetClosed
}
_, ok := s.set[string(cid.Hash())]
return ok, nil
}
func (s *MapMarkSet) Close() error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
s.set = nil
return nil
}
func (s *MapMarkSet) SetConcurrent() {
s.ts = true
}

13
blockstore/splitstore/markset_test.go
View File

@ -8,14 +8,23 @@ import (
"github.com/multiformats/go-multihash"
)
func TestBoltMarkSet(t *testing.T) {
testMarkSet(t, "bolt")
func TestMapMarkSet(t *testing.T) {
testMarkSet(t, "map")
}
func TestBloomMarkSet(t *testing.T) {
testMarkSet(t, "bloom")
}
func TestBadgerMarkSet(t *testing.T) {
bs := badgerMarkSetBatchSize
badgerMarkSetBatchSize = 1
t.Cleanup(func() {
badgerMarkSetBatchSize = bs
})
testMarkSet(t, "badger")
}
func testMarkSet(t *testing.T, lsType string) {
t.Helper()

1091
blockstore/splitstore/splitstore.go
View File

File diff suppressed because it is too large Load Diff

150
blockstore/splitstore/splitstore_check.go Normal file
View File

@ -0,0 +1,150 @@
package splitstore
import (
"fmt"
"os"
"path/filepath"
"sync/atomic"
"time"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
)
// performs an asynchronous health-check on the splitstore; results are appended to
// <splitstore-path>/check.txt
func (s *SplitStore) Check() error {
s.headChangeMx.Lock()
defer s.headChangeMx.Unlock()
// try to take compaction lock and inhibit compaction while the health-check is running
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
}
if s.compactionIndex == 0 {
atomic.StoreInt32(&s.compacting, 0)
return xerrors.Errorf("splitstore hasn't compacted yet; health check is not meaningful")
}
// check if we are actually closing first
if err := s.checkClosing(); err != nil {
atomic.StoreInt32(&s.compacting, 0)
return err
}
curTs := s.chain.GetHeaviestTipSet()
go func() {
defer atomic.StoreInt32(&s.compacting, 0)
log.Info("checking splitstore health")
start := time.Now()
err := s.doCheck(curTs)
if err != nil {
log.Errorf("error checking splitstore health: %s", err)
return
}
log.Infow("health check done", "took", time.Since(start))
}()
return nil
}
func (s *SplitStore) doCheck(curTs *types.TipSet) error {
currentEpoch := curTs.Height()
boundaryEpoch := currentEpoch - CompactionBoundary
outputPath := filepath.Join(s.path, "check.txt")
output, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return xerrors.Errorf("error opening check output file %s: %w", outputPath, err)
}
defer output.Close() //nolint:errcheck
write := func(format string, args ...interface{}) {
_, err := fmt.Fprintf(output, format+"\n", args...)
if err != nil {
log.Warnf("error writing check output: %s", err)
}
}
ts, _ := time.Now().MarshalText()
write("---------------------------------------------")
write("start check at %s", ts)
write("current epoch: %d", currentEpoch)
write("boundary epoch: %d", boundaryEpoch)
write("compaction index: %d", s.compactionIndex)
write("--")
var coldCnt, missingCnt int64
err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
has, err := s.hot.Has(c)
if err != nil {
return xerrors.Errorf("error checking hotstore: %w", err)
}
if has {
return nil
}
has, err = s.cold.Has(c)
if err != nil {
return xerrors.Errorf("error checking coldstore: %w", err)
}
if has {
coldCnt++
write("cold object reference: %s", c)
} else {
missingCnt++
write("missing object reference: %s", c)
return errStopWalk
}
return nil
})
if err != nil {
err = xerrors.Errorf("error walking chain: %w", err)
write("ERROR: %s", err)
return err
}
log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
write("--")
write("cold: %d missing: %d", coldCnt, missingCnt)
write("DONE")
return nil
}
// provides some basic information about the splitstore
func (s *SplitStore) Info() map[string]interface{} {
info := make(map[string]interface{})
info["base epoch"] = s.baseEpoch
info["warmup epoch"] = s.warmupEpoch
info["compactions"] = s.compactionIndex
sizer, ok := s.hot.(bstore.BlockstoreSize)
if ok {
size, err := sizer.Size()
if err != nil {
log.Warnf("error getting hotstore size: %s", err)
} else {
info["hotstore size"] = size
}
}
return info
}

1121
blockstore/splitstore/splitstore_compact.go Normal file
View File

File diff suppressed because it is too large Load Diff

114
blockstore/splitstore/splitstore_expose.go Normal file
View File

@ -0,0 +1,114 @@
package splitstore
import (
"context"
"errors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
)
type exposedSplitStore struct {
s *SplitStore
}
var _ bstore.Blockstore = (*exposedSplitStore)(nil)
func (s *SplitStore) Expose() bstore.Blockstore {
return &exposedSplitStore{s: s}
}
func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
return errors.New("DeleteBlock: operation not supported")
}
func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
return errors.New("DeleteMany: operation not supported")
}
func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
if isIdentiyCid(c) {
return true, nil
}
has, err := es.s.hot.Has(c)
if has || err != nil {
return has, err
}
return es.s.cold.Has(c)
}
func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
return nil, err
}
return blocks.NewBlockWithCid(data, c)
}
blk, err := es.s.hot.Get(c)
switch err {
case bstore.ErrNotFound:
return es.s.cold.Get(c)
default:
return blk, err
}
}
func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
return 0, err
}
return len(data), nil
}
size, err := es.s.hot.GetSize(c)
switch err {
case bstore.ErrNotFound:
return es.s.cold.GetSize(c)
default:
return size, err
}
}
func (es *exposedSplitStore) Put(blk blocks.Block) error {
return es.s.Put(blk)
}
func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
return es.s.PutMany(blks)
}
func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return es.s.AllKeysChan(ctx)
}
func (es *exposedSplitStore) HashOnRead(enabled bool) {}
func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
return err
}
return f(data)
}
err := es.s.hot.View(c, f)
switch err {
case bstore.ErrNotFound:
return es.s.cold.View(c, f)
default:
return err
}
}

35
blockstore/splitstore/splitstore_gc.go Normal file
View File

@ -0,0 +1,35 @@
package splitstore
import (
"fmt"
"time"
bstore "github.com/filecoin-project/lotus/blockstore"
)
func (s *SplitStore) gcHotstore() {
var opts []bstore.BlockstoreGCOption
if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
opts = append(opts, bstore.WithFullGC(true))
}
if err := s.gcBlockstore(s.hot, opts); err != nil {
log.Warnf("error garbage collecting hostore: %s", err)
}
}
func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
if gc, ok := b.(bstore.BlockstoreGC); ok {
log.Info("garbage collecting blockstore")
startGC := time.Now()
if err := gc.CollectGarbage(opts...); err != nil {
return err
}
log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
return nil
}
return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
}

292
blockstore/splitstore/splitstore_test.go
View File

@ -2,6 +2,7 @@ package splitstore
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
@ -13,6 +14,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
datastore "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
@ -21,22 +23,35 @@ import (
func init() {
CompactionThreshold = 5
CompactionCold = 1
CompactionBoundary = 2
WarmupBoundary = 0
logging.SetLogLevel("splitstore", "DEBUG")
}
func testSplitStore(t *testing.T, cfg *Config) {
chain := &mockChain{t: t}
// genesis
genBlock := mock.MkBlock(nil, 0, 0)
genTs := mock.TipSet(genBlock)
chain.push(genTs)
// the myriads of stores
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := blockstore.NewMemorySync()
cold := blockstore.NewMemorySync()
hot := newMockStore()
cold := newMockStore()
// this is necessary to avoid the garbage mock puts in the blocks
garbage := blocks.NewBlock([]byte{1, 2, 3})
err := cold.Put(garbage)
if err != nil {
t.Fatal(err)
}
// genesis
genBlock := mock.MkBlock(nil, 0, 0)
genBlock.Messages = garbage.Cid()
genBlock.ParentMessageReceipts = garbage.Cid()
genBlock.ParentStateRoot = garbage.Cid()
genBlock.Timestamp = uint64(time.Now().Unix())
genTs := mock.TipSet(genBlock)
chain.push(genTs)
// put the genesis block to cold store
blk, err := genBlock.ToStorageBlock()
@ -49,6 +64,20 @@ func testSplitStore(t *testing.T, cfg *Config) {
t.Fatal(err)
}
// create a garbage block that is protected with a rgistered protector
protected := blocks.NewBlock([]byte("protected!"))
err = hot.Put(protected)
if err != nil {
t.Fatal(err)
}
// and another one that is not protected
unprotected := blocks.NewBlock([]byte("unprotected!"))
err = hot.Put(unprotected)
if err != nil {
t.Fatal(err)
}
// open the splitstore
ss, err := Open("", ds, hot, cold, cfg)
if err != nil {
@ -56,18 +85,33 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
defer ss.Close() //nolint
// register our protector
ss.AddProtector(func(protect func(cid.Cid) error) error {
return protect(protected.Cid())
})
err = ss.Start(chain)
if err != nil {
t.Fatal(err)
}
// make some tipsets, but not enough to cause compaction
mkBlock := func(curTs *types.TipSet, i int) *types.TipSet {
mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
blk.Messages = garbage.Cid()
blk.ParentMessageReceipts = garbage.Cid()
blk.ParentStateRoot = stateRoot.Cid()
blk.Timestamp = uint64(time.Now().Unix())
sblk, err := blk.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = ss.Put(stateRoot)
if err != nil {
t.Fatal(err)
}
err = ss.Put(sblk)
if err != nil {
t.Fatal(err)
@ -78,18 +122,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
return ts
}
mkGarbageBlock := func(curTs *types.TipSet, i int) {
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
sblk, err := blk.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
err = ss.Put(sblk)
if err != nil {
t.Fatal(err)
}
}
waitForCompaction := func() {
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
@ -98,105 +130,101 @@ func testSplitStore(t *testing.T, cfg *Config) {
curTs := genTs
for i := 1; i < 5; i++ {
curTs = mkBlock(curTs, i)
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
curTs = mkBlock(curTs, i, stateRoot)
waitForCompaction()
}
mkGarbageBlock(genTs, 1)
// count objects in the cold and hot stores
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
countBlocks := func(bs blockstore.Blockstore) int {
count := 0
ch, err := bs.AllKeysChan(ctx)
if err != nil {
t.Fatal(err)
}
for range ch {
_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
count++
}
return nil
})
return count
}
coldCnt := countBlocks(cold)
hotCnt := countBlocks(hot)
if coldCnt != 1 {
t.Errorf("expected %d blocks, but got %d", 1, coldCnt)
if coldCnt != 2 {
t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
}
if hotCnt != 5 {
t.Errorf("expected %d blocks, but got %d", 5, hotCnt)
if hotCnt != 12 {
t.Errorf("expected %d blocks, but got %d", 12, hotCnt)
}
// trigger a compaction
for i := 5; i < 10; i++ {
curTs = mkBlock(curTs, i)
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
curTs = mkBlock(curTs, i, stateRoot)
waitForCompaction()
}
coldCnt = countBlocks(cold)
hotCnt = countBlocks(hot)
if !cfg.EnableFullCompaction {
if coldCnt != 5 {
t.Errorf("expected %d cold blocks, but got %d", 5, coldCnt)
}
if hotCnt != 5 {
t.Errorf("expected %d hot blocks, but got %d", 5, hotCnt)
}
if coldCnt != 6 {
t.Errorf("expected %d cold blocks, but got %d", 6, coldCnt)
}
if cfg.EnableFullCompaction && !cfg.EnableGC {
if coldCnt != 3 {
t.Errorf("expected %d cold blocks, but got %d", 3, coldCnt)
}
if hotCnt != 7 {
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
}
if hotCnt != 18 {
t.Errorf("expected %d hot blocks, but got %d", 18, hotCnt)
}
if cfg.EnableFullCompaction && cfg.EnableGC {
if coldCnt != 2 {
t.Errorf("expected %d cold blocks, but got %d", 2, coldCnt)
}
// ensure our protected block is still there
has, err := hot.Has(protected.Cid())
if err != nil {
t.Fatal(err)
}
if hotCnt != 7 {
t.Errorf("expected %d hot blocks, but got %d", 7, hotCnt)
}
if !has {
t.Fatal("protected block is missing from hotstore")
}
// ensure our unprotected block is in the coldstore now
has, err = hot.Has(unprotected.Cid())
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unprotected block is still in hotstore")
}
has, err = cold.Has(unprotected.Cid())
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("unprotected block is missing from coldstore")
}
// Make sure we can revert without panicking.
chain.revert(2)
}
func TestSplitStoreSimpleCompaction(t *testing.T) {
testSplitStore(t, &Config{TrackingStoreType: "mem"})
func TestSplitStoreCompaction(t *testing.T) {
testSplitStore(t, &Config{MarkSetType: "map"})
}
func TestSplitStoreFullCompactionWithoutGC(t *testing.T) {
testSplitStore(t, &Config{
TrackingStoreType: "mem",
EnableFullCompaction: true,
})
}
func TestSplitStoreFullCompactionWithGC(t *testing.T) {
testSplitStore(t, &Config{
TrackingStoreType: "mem",
EnableFullCompaction: true,
EnableGC: true,
func TestSplitStoreCompactionWithBadger(t *testing.T) {
bs := badgerMarkSetBatchSize
badgerMarkSetBatchSize = 1
t.Cleanup(func() {
badgerMarkSetBatchSize = bs
})
testSplitStore(t, &Config{MarkSetType: "badger"})
}
type mockChain struct {
t testing.TB
sync.Mutex
genesis *types.BlockHeader
tipsets []*types.TipSet
listener func(revert []*types.TipSet, apply []*types.TipSet) error
}
@ -204,6 +232,9 @@ type mockChain struct {
func (c *mockChain) push(ts *types.TipSet) {
c.Lock()
c.tipsets = append(c.tipsets, ts)
if c.genesis == nil {
c.genesis = ts.Blocks()[0]
}
c.Unlock()
if c.listener != nil {
@ -242,7 +273,7 @@ func (c *mockChain) GetTipsetByHeight(_ context.Context, epoch abi.ChainEpoch, _
return nil, fmt.Errorf("bad epoch %d", epoch)
}
return c.tipsets[iEpoch-1], nil
return c.tipsets[iEpoch], nil
}
func (c *mockChain) GetHeaviestTipSet() *types.TipSet {
@ -256,24 +287,105 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
c.listener = change
}
func (c *mockChain) WalkSnapshot(_ context.Context, ts *types.TipSet, epochs abi.ChainEpoch, _ bool, _ bool, f func(cid.Cid) error) error {
c.Lock()
defer c.Unlock()
type mockStore struct {
mx sync.Mutex
set map[cid.Cid]blocks.Block
}
start := int(ts.Height()) - 1
end := start - int(epochs)
if end < 0 {
end = -1
func newMockStore() *mockStore {
return &mockStore{set: make(map[cid.Cid]blocks.Block)}
}
func (b *mockStore) Has(cid cid.Cid) (bool, error) {
b.mx.Lock()
defer b.mx.Unlock()
_, ok := b.set[cid]
return ok, nil
}
func (b *mockStore) HashOnRead(hor bool) {}
func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
b.mx.Lock()
defer b.mx.Unlock()
blk, ok := b.set[cid]
if !ok {
return nil, blockstore.ErrNotFound
}
for i := start; i > end; i-- {
ts := c.tipsets[i]
for _, cid := range ts.Cids() {
err := f(cid)
if err != nil {
return err
}
}
return blk, nil
}
func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
blk, err := b.Get(cid)
if err != nil {
return 0, err
}
return len(blk.RawData()), nil
}
func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
blk, err := b.Get(cid)
if err != nil {
return err
}
return f(blk.RawData())
}
func (b *mockStore) Put(blk blocks.Block) error {
b.mx.Lock()
defer b.mx.Unlock()
b.set[blk.Cid()] = blk
return nil
}
func (b *mockStore) PutMany(blks []blocks.Block) error {
b.mx.Lock()
defer b.mx.Unlock()
for _, blk := range blks {
b.set[blk.Cid()] = blk
}
return nil
}
func (b *mockStore) DeleteBlock(cid cid.Cid) error {
b.mx.Lock()
defer b.mx.Unlock()
delete(b.set, cid)
return nil
}
func (b *mockStore) DeleteMany(cids []cid.Cid) error {
b.mx.Lock()
defer b.mx.Unlock()
for _, c := range cids {
delete(b.set, c)
}
return nil
}
func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, errors.New("not implemented")
}
func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
b.mx.Lock()
defer b.mx.Unlock()
for c := range b.set {
err := f(c)
if err != nil {
return err
}
}
return nil
}
func (b *mockStore) Close() error {
return nil
}

67
blockstore/splitstore/splitstore_util.go Normal file
View File

@ -0,0 +1,67 @@
package splitstore
import (
"encoding/binary"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
)
func epochToBytes(epoch abi.ChainEpoch) []byte {
return uint64ToBytes(uint64(epoch))
}
func bytesToEpoch(buf []byte) abi.ChainEpoch {
return abi.ChainEpoch(bytesToUint64(buf))
}
func int64ToBytes(i int64) []byte {
return uint64ToBytes(uint64(i))
}
func bytesToInt64(buf []byte) int64 {
return int64(bytesToUint64(buf))
}
func uint64ToBytes(i uint64) []byte {
buf := make([]byte, 16)
n := binary.PutUvarint(buf, i)
return buf[:n]
}
func bytesToUint64(buf []byte) uint64 {
i, _ := binary.Uvarint(buf)
return i
}
func isUnitaryObject(c cid.Cid) bool {
pre := c.Prefix()
switch pre.Codec {
case cid.FilCommitmentSealed, cid.FilCommitmentUnsealed:
return true
default:
return pre.MhType == mh.IDENTITY
}
}
func isIdentiyCid(c cid.Cid) bool {
return c.Prefix().MhType == mh.IDENTITY
}
func decodeIdentityCid(c cid.Cid) ([]byte, error) {
dmh, err := mh.Decode(c.Hash())
if err != nil {
return nil, xerrors.Errorf("error decoding identity cid %s: %w", c, err)
}
// sanity check
if dmh.Code != mh.IDENTITY {
return nil, xerrors.Errorf("error decoding identity cid %s: hash type is not identity", c)
}
return dmh.Digest, nil
}

137
blockstore/splitstore/splitstore_warmup.go Normal file
View File

@ -0,0 +1,137 @@
package splitstore
import (
"sync/atomic"
"time"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
var (
// WarmupBoundary is the number of epochs to load state during warmup.
WarmupBoundary = build.Finality
)
// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
// this is necessary when we sync from a snapshot or when we enable the splitstore
// on top of an existing blockstore (which becomes the coldstore).
func (s *SplitStore) warmup(curTs *types.TipSet) error {
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
return xerrors.Errorf("error locking compaction")
}
go func() {
defer atomic.StoreInt32(&s.compacting, 0)
log.Info("warming up hotstore")
start := time.Now()
err := s.doWarmup(curTs)
if err != nil {
log.Errorf("error warming up hotstore: %s", err)
return
}
log.Infow("warm up done", "took", time.Since(start))
}()
return nil
}
// the actual warmup procedure; it walks the chain loading all state roots at the boundary
// and headers all the way up to genesis.
// objects are written in batches so as to minimize overhead.
func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
var boundaryEpoch abi.ChainEpoch
epoch := curTs.Height()
if WarmupBoundary < epoch {
boundaryEpoch = epoch - WarmupBoundary
}
batchHot := make([]blocks.Block, 0, batchSize)
count := int64(0)
xcount := int64(0)
missing := int64(0)
err := s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
count++
has, err := s.hot.Has(c)
if err != nil {
return err
}
if has {
return nil
}
blk, err := s.cold.Get(c)
if err != nil {
if err == bstore.ErrNotFound {
missing++
return errStopWalk
}
return err
}
xcount++
batchHot = append(batchHot, blk)
if len(batchHot) == batchSize {
err = s.hot.PutMany(batchHot)
if err != nil {
return err
}
batchHot = batchHot[:0]
}
return nil
})
if err != nil {
return err
}
if len(batchHot) > 0 {
err = s.hot.PutMany(batchHot)
if err != nil {
return err
}
}
log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
s.markSetSize = count + count>>2 // overestimate a bit
err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
if err != nil {
log.Warnf("error saving mark set size: %s", err)
}
// save the warmup epoch
err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
if err != nil {
return xerrors.Errorf("error saving warm up epoch: %w", err)
}
s.mx.Lock()
s.warmupEpoch = epoch
s.mx.Unlock()
// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
if err != nil {
return xerrors.Errorf("error saving compaction index: %w", err)
}
return nil
}

109
blockstore/splitstore/tracking.go
View File

@ -1,109 +0,0 @@
package splitstore
import (
"path/filepath"
"sync"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
cid "github.com/ipfs/go-cid"
)
// TrackingStore is a persistent store that tracks blocks that are added
// to the hotstore, tracking the epoch at which they are written.
type TrackingStore interface {
Put(cid.Cid, abi.ChainEpoch) error
PutBatch([]cid.Cid, abi.ChainEpoch) error
Get(cid.Cid) (abi.ChainEpoch, error)
Delete(cid.Cid) error
DeleteBatch([]cid.Cid) error
ForEach(func(cid.Cid, abi.ChainEpoch) error) error
Sync() error
Close() error
}
// OpenTrackingStore opens a tracking store of the specified type in the
// specified path.
func OpenTrackingStore(path string, ttype string) (TrackingStore, error) {
switch ttype {
case "", "bolt":
return OpenBoltTrackingStore(filepath.Join(path, "tracker.bolt"))
case "mem":
return NewMemTrackingStore(), nil
default:
return nil, xerrors.Errorf("unknown tracking store type %s", ttype)
}
}
// NewMemTrackingStore creates an in-memory tracking store.
// This is only useful for test or situations where you don't want to open the
// real tracking store (eg concurrent read only access on a node's datastore)
func NewMemTrackingStore() *MemTrackingStore {
return &MemTrackingStore{tab: make(map[cid.Cid]abi.ChainEpoch)}
}
// MemTrackingStore is a simple in-memory tracking store
type MemTrackingStore struct {
sync.Mutex
tab map[cid.Cid]abi.ChainEpoch
}
var _ TrackingStore = (*MemTrackingStore)(nil)
func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
s.Lock()
defer s.Unlock()
s.tab[cid] = epoch
return nil
}
func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
s.Lock()
defer s.Unlock()
for _, cid := range cids {
s.tab[cid] = epoch
}
return nil
}
func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error) {
s.Lock()
defer s.Unlock()
epoch, ok := s.tab[cid]
if ok {
return epoch, nil
}
return 0, xerrors.Errorf("missing tracking epoch for %s", cid)
}
func (s *MemTrackingStore) Delete(cid cid.Cid) error {
s.Lock()
defer s.Unlock()
delete(s.tab, cid)
return nil
}
func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error {
s.Lock()
defer s.Unlock()
for _, cid := range cids {
delete(s.tab, cid)
}
return nil
}
func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
s.Lock()
defer s.Unlock()
for cid, epoch := range s.tab {
err := f(cid, epoch)
if err != nil {
return err
}
}
return nil
}
func (s *MemTrackingStore) Sync() error { return nil }
func (s *MemTrackingStore) Close() error { return nil }

120
blockstore/splitstore/tracking_bolt.go
View File

@ -1,120 +0,0 @@
package splitstore
import (
"time"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bolt "go.etcd.io/bbolt"
"github.com/filecoin-project/go-state-types/abi"
)
type BoltTrackingStore struct {
db *bolt.DB
bucketId []byte
}
var _ TrackingStore = (*BoltTrackingStore)(nil)
func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error) {
opts := &bolt.Options{
Timeout: 1 * time.Second,
NoSync: true,
}
db, err := bolt.Open(path, 0644, opts)
if err != nil {
return nil, err
}
bucketId := []byte("tracker")
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucketId)
if err != nil {
return xerrors.Errorf("error creating bolt db bucket %s: %w", string(bucketId), err)
}
return nil
})
if err != nil {
_ = db.Close()
return nil, err
}
return &BoltTrackingStore{db: db, bucketId: bucketId}, nil
}
func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error {
val := epochToBytes(epoch)
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Put(cid.Hash(), val)
})
}
func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error {
val := epochToBytes(epoch)
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
for _, cid := range cids {
err := b.Put(cid.Hash(), val)
if err != nil {
return err
}
}
return nil
})
}
func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error) {
err = s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
val := b.Get(cid.Hash())
if val == nil {
return xerrors.Errorf("missing tracking epoch for %s", cid)
}
epoch = bytesToEpoch(val)
return nil
})
return epoch, err
}
func (s *BoltTrackingStore) Delete(cid cid.Cid) error {
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.Delete(cid.Hash())
})
}
func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error {
return s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
for _, cid := range cids {
err := b.Delete(cid.Hash())
if err != nil {
return xerrors.Errorf("error deleting %s", cid)
}
}
return nil
})
}
func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error {
return s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(s.bucketId)
return b.ForEach(func(k, v []byte) error {
cid := cid.NewCidV1(cid.Raw, k)
epoch := bytesToEpoch(v)
return f(cid, epoch)
})
})
}
func (s *BoltTrackingStore) Sync() error {
return s.db.Sync()
}
func (s *BoltTrackingStore) Close() error {
return s.db.Close()
}

130
blockstore/splitstore/tracking_test.go
View File

@ -1,130 +0,0 @@
package splitstore
import (
"io/ioutil"
"testing"
cid "github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
)
func TestBoltTrackingStore(t *testing.T) {
testTrackingStore(t, "bolt")
}
func testTrackingStore(t *testing.T, tsType string) {
t.Helper()
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s TrackingStore, cid cid.Cid, epoch abi.ChainEpoch) {
val, err := s.Get(cid)
if err != nil {
t.Fatal(err)
}
if val != epoch {
t.Fatal("epoch mismatch")
}
}
mustNotHave := func(s TrackingStore, cid cid.Cid) {
_, err := s.Get(cid)
if err == nil {
t.Fatal("expected error")
}
}
path, err := ioutil.TempDir("", "snoop-test.*")
if err != nil {
t.Fatal(err)
}
s, err := OpenTrackingStore(path, tsType)
if err != nil {
t.Fatal(err)
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
s.Put(k1, 1) //nolint
s.Put(k2, 2) //nolint
s.Put(k3, 3) //nolint
s.Put(k4, 4) //nolint
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.Delete(k1) // nolint
s.Delete(k2) // nolint
mustNotHave(s, k1)
mustNotHave(s, k2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.PutBatch([]cid.Cid{k1}, 1) //nolint
s.PutBatch([]cid.Cid{k2}, 2) //nolint
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
allKeys := map[string]struct{}{
k1.String(): {},
k2.String(): {},
k3.String(): {},
k4.String(): {},
}
err = s.ForEach(func(k cid.Cid, _ abi.ChainEpoch) error {
_, ok := allKeys[k.String()]
if !ok {
t.Fatal("unexpected key")
}
delete(allKeys, k.String())
return nil
})
if err != nil {
t.Fatal(err)
}
if len(allKeys) != 0 {
t.Fatal("not all keys were returned")
}
// no close and reopen and ensure the keys still exist
err = s.Close()
if err != nil {
t.Fatal(err)
}
s, err = OpenTrackingStore(path, tsType)
if err != nil {
t.Fatal(err)
}
mustHave(s, k1, 1)
mustHave(s, k2, 2)
mustHave(s, k3, 3)
mustHave(s, k4, 4)
s.Close() //nolint:errcheck
}

4
build/bootstrap/interopnet.pi
View File

@ -1,2 +1,2 @@
/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWN86wA54r3v9M8bBYbc1vK9W1ehHDxVGPRaoeUYuXF8R7
/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWNZ41kev8mtBZgWe43qam1VX9pJyf87jnaisQP2urZZ2M
/dns4/bootstrap-0.interop.fildev.network/tcp/1347/p2p/12D3KooWLGPq9JL1xwL6gHok7HSNxtK1Q5kyfg4Hk69ifRPghn4i
/dns4/bootstrap-1.interop.fildev.network/tcp/1347/p2p/12D3KooWFYS1f31zafv8mqqYu8U3hEqYvaZ6avWzYU3BmZdpyH3h

BIN
build/genesis/interopnet.car
View File

Binary file not shown.

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

17
build/params_2k.go
View File

@ -28,18 +28,19 @@ var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
var UpgradeLiftoffHeight = abi.ChainEpoch(-6)
var UpgradeKumquatHeight = abi.ChainEpoch(-7)
var UpgradeCalicoHeight = abi.ChainEpoch(-8)
var UpgradePersianHeight = abi.ChainEpoch(-9)
var UpgradeOrangeHeight = abi.ChainEpoch(-10)
var UpgradeClausHeight = abi.ChainEpoch(-11)
var UpgradePricelistOopsHeight = abi.ChainEpoch(-8)
var UpgradeCalicoHeight = abi.ChainEpoch(-9)
var UpgradePersianHeight = abi.ChainEpoch(-10)
var UpgradeOrangeHeight = abi.ChainEpoch(-11)
var UpgradeClausHeight = abi.ChainEpoch(-12)
var UpgradeTrustHeight = abi.ChainEpoch(-12)
var UpgradeTrustHeight = abi.ChainEpoch(-13)
var UpgradeNorwegianHeight = abi.ChainEpoch(-13)
var UpgradeNorwegianHeight = abi.ChainEpoch(-14)
var UpgradeTurboHeight = abi.ChainEpoch(-14)
var UpgradeTurboHeight = abi.ChainEpoch(-15)
var UpgradeHyperdriveHeight = abi.ChainEpoch(-15)
var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,

1
build/params_butterfly.go
View File

@ -28,6 +28,7 @@ var UpgradeAssemblyHeight = abi.ChainEpoch(30)
const UpgradeTapeHeight = 60
const UpgradeLiftoffHeight = -5
const UpgradeKumquatHeight = 90
const UpgradePricelistOopsHeight = 119
const UpgradeCalicoHeight = 120
const UpgradePersianHeight = 150
const UpgradeClausHeight = 180

2
build/params_calibnet.go
View File

@ -33,6 +33,8 @@ const UpgradeLiftoffHeight = -5
const UpgradeKumquatHeight = 90
const UpgradePricelistOopsHeight = 119
const UpgradeCalicoHeight = 120
const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)

17
build/params_interop.go
View File

@ -31,18 +31,19 @@ var UpgradeAssemblyHeight = abi.ChainEpoch(-5)
var UpgradeLiftoffHeight = abi.ChainEpoch(-6)
var UpgradeKumquatHeight = abi.ChainEpoch(-7)
var UpgradeCalicoHeight = abi.ChainEpoch(-8)
var UpgradePersianHeight = abi.ChainEpoch(-9)
var UpgradeOrangeHeight = abi.ChainEpoch(-10)
var UpgradeClausHeight = abi.ChainEpoch(-11)
var UpgradePricelistOopsHeight = abi.ChainEpoch(-8)
var UpgradeCalicoHeight = abi.ChainEpoch(-9)
var UpgradePersianHeight = abi.ChainEpoch(-10)
var UpgradeOrangeHeight = abi.ChainEpoch(-11)
var UpgradeClausHeight = abi.ChainEpoch(-12)
var UpgradeTrustHeight = abi.ChainEpoch(-12)
var UpgradeTrustHeight = abi.ChainEpoch(-13)
var UpgradeNorwegianHeight = abi.ChainEpoch(-13)
var UpgradeNorwegianHeight = abi.ChainEpoch(-14)
var UpgradeTurboHeight = abi.ChainEpoch(-14)
var UpgradeTurboHeight = abi.ChainEpoch(-15)
var UpgradeHyperdriveHeight = abi.ChainEpoch(-15)
var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,

1
build/params_mainnet.go
View File

@ -45,6 +45,7 @@ const UpgradeLiftoffHeight = 148888
const UpgradeKumquatHeight = 170000
const UpgradePricelistOopsHeight = 265199
const UpgradeCalicoHeight = 265200
const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 60)

1
build/params_nerpanet.go
View File

@ -32,6 +32,7 @@ const UpgradeTapeHeight = 60
const UpgradeKumquatHeight = 90
const UpgradePricelistOopsHeight = 99
const UpgradeCalicoHeight = 100
const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)

31
build/params_testground.go
View File

@ -82,21 +82,22 @@ var (
UpgradeBreezeHeight abi.ChainEpoch = -1
BreezeGasTampingDuration abi.ChainEpoch = 0
UpgradeSmokeHeight abi.ChainEpoch = -1
UpgradeIgnitionHeight abi.ChainEpoch = -2
UpgradeRefuelHeight abi.ChainEpoch = -3
UpgradeTapeHeight abi.ChainEpoch = -4
UpgradeAssemblyHeight abi.ChainEpoch = 10
UpgradeLiftoffHeight abi.ChainEpoch = -5
UpgradeKumquatHeight abi.ChainEpoch = -6
UpgradeCalicoHeight abi.ChainEpoch = -7
UpgradePersianHeight abi.ChainEpoch = -8
UpgradeOrangeHeight abi.ChainEpoch = -9
UpgradeClausHeight abi.ChainEpoch = -10
UpgradeTrustHeight abi.ChainEpoch = -11
UpgradeNorwegianHeight abi.ChainEpoch = -12
UpgradeTurboHeight abi.ChainEpoch = -13
UpgradeHyperdriveHeight abi.ChainEpoch = -13
UpgradeSmokeHeight abi.ChainEpoch = -1
UpgradeIgnitionHeight abi.ChainEpoch = -2
UpgradeRefuelHeight abi.ChainEpoch = -3
UpgradeTapeHeight abi.ChainEpoch = -4
UpgradeAssemblyHeight abi.ChainEpoch = 10
UpgradeLiftoffHeight abi.ChainEpoch = -5
UpgradeKumquatHeight abi.ChainEpoch = -6
UpgradePricelistOopsHeight abi.ChainEpoch = -7
UpgradeCalicoHeight abi.ChainEpoch = -8
UpgradePersianHeight abi.ChainEpoch = -9
UpgradeOrangeHeight abi.ChainEpoch = -10
UpgradeClausHeight abi.ChainEpoch = -11
UpgradeTrustHeight abi.ChainEpoch = -12
UpgradeNorwegianHeight abi.ChainEpoch = -13
UpgradeTurboHeight abi.ChainEpoch = -14
UpgradeHyperdriveHeight abi.ChainEpoch = -15
DrandSchedule = map[abi.ChainEpoch]DrandEnum{
0: DrandMainnet,

2
build/version.go
View File

@ -34,7 +34,7 @@ func buildType() string {
}
// BuildVersion is the local build version, set by build system
const BuildVersion = "1.11.0"
const BuildVersion = "1.11.1-dev"
func UserVersion() string {
if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

1
chain/actors/builtin/miner/actor.go.template
View File

@ -105,6 +105,7 @@ type State interface {
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
// count if there aren't enough).
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
GetAllocatedSectors() (*bitfield.BitField, error)
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
GetProvingPeriodStart() (abi.ChainEpoch, error)

1
chain/actors/builtin/miner/miner.go
View File

@ -164,6 +164,7 @@ type State interface {
// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
// count if there aren't enough).
UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
GetAllocatedSectors() (*bitfield.BitField, error)
// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
GetProvingPeriodStart() (abi.ChainEpoch, error)

9
chain/actors/builtin/miner/state.go.template
View File

@ -318,6 +318,15 @@ func (s *state{{.v}}) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, e
return sectors, nil
}
func (s *state{{.v}}) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state{{.v}}) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

9
chain/actors/builtin/miner/v0.go
View File

@ -311,6 +311,15 @@ func (s *state0) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
return sectors, nil
}
func (s *state0) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state0) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

9
chain/actors/builtin/miner/v2.go
View File

@ -309,6 +309,15 @@ func (s *state2) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
return sectors, nil
}
func (s *state2) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state2) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

9
chain/actors/builtin/miner/v3.go
View File

@ -311,6 +311,15 @@ func (s *state3) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
return sectors, nil
}
func (s *state3) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state3) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

9
chain/actors/builtin/miner/v4.go
View File

@ -311,6 +311,15 @@ func (s *state4) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
return sectors, nil
}
func (s *state4) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state4) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

9
chain/actors/builtin/miner/v5.go
View File

@ -311,6 +311,15 @@ func (s *state5) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
return sectors, nil
}
func (s *state5) GetAllocatedSectors() (*bitfield.BitField, error) {
var allocatedSectors bitfield.BitField
if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
return nil, err
}
return &allocatedSectors, nil
}
func (s *state5) LoadDeadline(idx uint64) (Deadline, error) {
dls, err := s.State.LoadDeadlines(s.store)
if err != nil {

27
chain/actors/policy/policy.go
View File

@ -196,6 +196,33 @@ func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) ab
}
}
// SetProviderCollateralSupplyTarget sets the percentage of normalized circulating
// supply that must be covered by provider collateral in a deal. This should
// only be used for testing.
func SetProviderCollateralSupplyTarget(num, denom big.Int) {
market2.ProviderCollateralSupplyTarget = builtin2.BigFrac{
Numerator: num,
Denominator: denom,
}
market3.ProviderCollateralSupplyTarget = builtin3.BigFrac{
Numerator: num,
Denominator: denom,
}
market4.ProviderCollateralSupplyTarget = builtin4.BigFrac{
Numerator: num,
Denominator: denom,
}
market5.ProviderCollateralSupplyTarget = builtin5.BigFrac{
Numerator: num,
Denominator: denom,
}
}
func DealProviderCollateralBounds(
size abi.PaddedPieceSize, verified bool,
rawBytePower, qaPower, baselinePower abi.StoragePower,

14
chain/actors/policy/policy.go.template
View File

@ -132,6 +132,20 @@ func GetMaxProveCommitDuration(ver actors.Version, t abi.RegisteredSealProof) ab
}
}
// SetProviderCollateralSupplyTarget sets the percentage of normalized circulating
// supply that must be covered by provider collateral in a deal. This should
// only be used for testing.
func SetProviderCollateralSupplyTarget(num, denom big.Int) {
{{range .versions}}
{{if (ge . 2)}}
market{{.}}.ProviderCollateralSupplyTarget = builtin{{.}}.BigFrac{
Numerator: num,
Denominator: denom,
}
{{end}}
{{end}}
}
func DealProviderCollateralBounds(
size abi.PaddedPieceSize, verified bool,
rawBytePower, qaPower, baselinePower abi.StoragePower,

2
chain/actors/version.go
View File

@ -25,7 +25,7 @@ func VersionForNetwork(version network.Version) Version {
switch version {
case network.Version0, network.Version1, network.Version2, network.Version3:
return Version0
case network.Version4, network.Version5, network.Version6, network.Version7, network.Version8, network.Version9:
case network.Version4, network.Version5, network.Version6, network.Version6AndAHalf, network.Version7, network.Version8, network.Version9:
return Version2
case network.Version10, network.Version11:
return Version3

38
chain/events/events_called.go
View File

@ -5,6 +5,9 @@ import (
"math"
"sync"
"github.com/filecoin-project/lotus/api"
lru "github.com/hashicorp/golang-lru"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/go-state-types/abi"
@ -464,14 +467,20 @@ type messageEvents struct {
lk sync.RWMutex
matchers map[triggerID]MsgMatchFunc
blockMsgLk sync.Mutex
blockMsgCache *lru.ARCCache
}
func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs EventAPI) messageEvents {
blsMsgCache, _ := lru.NewARC(500)
return messageEvents{
ctx: ctx,
cs: cs,
hcAPI: hcAPI,
matchers: make(map[triggerID]MsgMatchFunc),
ctx: ctx,
cs: cs,
hcAPI: hcAPI,
matchers: make(map[triggerID]MsgMatchFunc),
blockMsgLk: sync.Mutex{},
blockMsgCache: blsMsgCache,
}
}
@ -515,14 +524,21 @@ func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Mes
seen := map[cid.Cid]struct{}{}
for _, tsb := range ts.Blocks() {
msgs, err := me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid())
if err != nil {
log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err)
// this is quite bad, but probably better than missing all the other updates
continue
me.blockMsgLk.Lock()
msgsI, ok := me.blockMsgCache.Get(tsb.Cid())
var err error
if !ok {
msgsI, err = me.cs.ChainGetBlockMessages(context.TODO(), tsb.Cid())
if err != nil {
log.Errorf("messagesForTs MessagesForBlock failed (ts.H=%d, Bcid:%s, B.Mcid:%s): %s", ts.Height(), tsb.Cid(), tsb.Messages, err)
// this is quite bad, but probably better than missing all the other updates
me.blockMsgLk.Unlock()
continue
}
me.blockMsgCache.Add(tsb.Cid(), msgsI)
}
me.blockMsgLk.Unlock()
msgs := msgsI.(*api.BlockMessages)
for _, m := range msgs.BlsMessages {
_, ok := seen[m.Cid()]
if ok {

190
chain/events/events_test.go
View File

@ -6,6 +6,8 @@ import (
"sync"
"testing"
"gotest.tools/assert"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
@ -44,25 +46,43 @@ type fakeCS struct {
tipsets map[types.TipSetKey]*types.TipSet
sub func(rev, app []*types.TipSet)
callNumberLk sync.Mutex
callNumber map[string]int
}
func (fcs *fakeCS) ChainHead(ctx context.Context) (*types.TipSet, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["ChainHead"] = fcs.callNumber["ChainHead"] + 1
panic("implement me")
}
func (fcs *fakeCS) ChainGetTipSet(ctx context.Context, key types.TipSetKey) (*types.TipSet, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["ChainGetTipSet"] = fcs.callNumber["ChainGetTipSet"] + 1
return fcs.tipsets[key], nil
}
func (fcs *fakeCS) StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*api.MsgLookup, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["StateSearchMsg"] = fcs.callNumber["StateSearchMsg"] + 1
return nil, nil
}
func (fcs *fakeCS) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["StateGetActor"] = fcs.callNumber["StateGetActor"] + 1
panic("Not Implemented")
}
func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["ChainGetTipSetByHeight"] = fcs.callNumber["ChainGetTipSetByHeight"] + 1
panic("Not Implemented")
}
@ -113,6 +133,10 @@ func (fcs *fakeCS) makeTs(t *testing.T, parents []cid.Cid, h abi.ChainEpoch, msg
}
func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["ChainNotify"] = fcs.callNumber["ChainNotify"] + 1
out := make(chan []*api.HeadChange, 1)
best, err := fcs.tsc.best()
if err != nil {
@ -143,6 +167,9 @@ func (fcs *fakeCS) ChainNotify(context.Context) (<-chan []*api.HeadChange, error
}
func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api.BlockMessages, error) {
fcs.callNumberLk.Lock()
defer fcs.callNumberLk.Unlock()
fcs.callNumber["ChainGetBlockMessages"] = fcs.callNumber["ChainGetBlockMessages"] + 1
messages, ok := fcs.blkMsgs[blk]
if !ok {
return &api.BlockMessages{}, nil
@ -152,8 +179,8 @@ func (fcs *fakeCS) ChainGetBlockMessages(ctx context.Context, blk cid.Cid) (*api
if !ok {
return &api.BlockMessages{}, nil
}
return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs}, nil
return &api.BlockMessages{BlsMessages: ms.bmsgs, SecpkMessages: ms.smsgs}, nil
}
func (fcs *fakeCS) fakeMsgs(m fakeMsg) cid.Cid {
@ -233,9 +260,10 @@ var _ EventAPI = &fakeCS{}
func TestAt(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -298,9 +326,10 @@ func TestAt(t *testing.T) {
func TestAtDoubleTrigger(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -340,9 +369,10 @@ func TestAtDoubleTrigger(t *testing.T) {
func TestAtNullTrigger(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -374,9 +404,10 @@ func TestAtNullTrigger(t *testing.T) {
func TestAtNullConf(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -413,9 +444,10 @@ func TestAtNullConf(t *testing.T) {
func TestAtStart(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -447,9 +479,10 @@ func TestAtStart(t *testing.T) {
func TestAtStartConfidence(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -477,9 +510,10 @@ func TestAtStartConfidence(t *testing.T) {
func TestAtChained(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -511,9 +545,10 @@ func TestAtChained(t *testing.T) {
func TestAtChainedConfidence(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -545,9 +580,10 @@ func TestAtChainedConfidence(t *testing.T) {
func TestAtChainedConfidenceNull(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
t: t,
h: 1,
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -583,9 +619,10 @@ func TestCalled(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -795,9 +832,10 @@ func TestCalledTimeout(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -835,9 +873,10 @@ func TestCalledTimeout(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
callNumber: map[string]int{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -869,9 +908,10 @@ func TestCalledOrder(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -932,9 +972,10 @@ func TestCalledNull(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -997,9 +1038,10 @@ func TestRemoveTriggersOnMessage(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1087,9 +1129,10 @@ func TestStateChanged(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1175,9 +1218,10 @@ func TestStateChangedRevert(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1253,9 +1297,10 @@ func TestStateChangedTimeout(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
callNumber: map[string]int{},
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1293,9 +1338,10 @@ func TestStateChangedTimeout(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
callNumber: map[string]int{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1329,9 +1375,10 @@ func TestCalledMultiplePerEpoch(t *testing.T) {
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
callNumber: map[string]int{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
@ -1382,3 +1429,24 @@ func TestCalledMultiplePerEpoch(t *testing.T) {
fcs.advance(9, 1, nil)
}
func TestCachedSameBlock(t *testing.T) {
fcs := &fakeCS{
t: t,
h: 1,
msgs: map[cid.Cid]fakeMsg{},
blkMsgs: map[cid.Cid]cid.Cid{},
callNumber: map[string]int{},
tsc: newTSCache(2*build.ForkLengthThreshold, nil),
}
require.NoError(t, fcs.tsc.add(fcs.makeTs(t, nil, 1, dummyCid)))
_ = NewEvents(context.Background(), fcs)
fcs.advance(0, 10, map[int]cid.Cid{})
assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 20, "expect call ChainGetBlockMessages %d but got ", 20, fcs.callNumber["ChainGetBlockMessages"])
fcs.advance(5, 10, map[int]cid.Cid{})
assert.Assert(t, fcs.callNumber["ChainGetBlockMessages"] == 30, "expect call ChainGetBlockMessages %d but got ", 30, fcs.callNumber["ChainGetBlockMessages"])
}

2
chain/events/state/predicates_test.go
View File

@ -21,7 +21,7 @@ import (
market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
adt2 "github.com/filecoin-project/specs-actors/v2/actors/util/adt"
tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
tutils "github.com/filecoin-project/specs-actors/v5/support/testing"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"

4
chain/gen/gen.go
View File

@ -233,7 +233,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
return nil, xerrors.Errorf("make genesis block failed: %w", err)
}
cs := store.NewChainStore(bs, bs, ds, sys, j)
cs := store.NewChainStore(bs, bs, ds, j)
genfb := &types.FullBlock{Header: genb.Genesis}
gents := store.NewFullTipSet([]*types.FullBlock{genfb})
@ -247,7 +247,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
mgen[genesis2.MinerAddress(uint64(i))] = &wppProvider{}
}
sm, err := stmgr.NewStateManagerWithUpgradeSchedule(cs, us)
sm, err := stmgr.NewStateManagerWithUpgradeSchedule(cs, sys, us)
if err != nil {
return nil, xerrors.Errorf("initing stmgr: %w", err)
}

10
chain/gen/genesis/genesis.go
View File

@ -471,7 +471,7 @@ func createMultisigAccount(ctx context.Context, cst cbor.IpldStore, state *state
return nil
}
func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot cid.Cid, template genesis.Template, keyIDs map[address.Address]address.Address, nv network.Version) (cid.Cid, error) {
func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, sys vm.SyscallBuilder, stateroot cid.Cid, template genesis.Template, keyIDs map[address.Address]address.Address, nv network.Version) (cid.Cid, error) {
verifNeeds := make(map[address.Address]abi.PaddedPieceSize)
var sum abi.PaddedPieceSize
@ -480,7 +480,7 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, stateroot ci
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.StateBlockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
Syscalls: mkFakedSigSyscalls(sys),
CircSupplyCalc: nil,
NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
return nv
@ -559,15 +559,15 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto
}
// temp chainstore
cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), sys, j)
cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), j)
// Verify PreSealed Data
stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs, template.NetworkVersion)
stateroot, err = VerifyPreSealedData(ctx, cs, sys, stateroot, template, keyIDs, template.NetworkVersion)
if err != nil {
return nil, xerrors.Errorf("failed to verify presealed data: %w", err)
}
stateroot, err = SetupStorageMiners(ctx, cs, stateroot, template.Miners, template.NetworkVersion)
stateroot, err = SetupStorageMiners(ctx, cs, sys, stateroot, template.Miners, template.NetworkVersion)
if err != nil {
return nil, xerrors.Errorf("setup miners failed: %w", err)
}

4
chain/gen/genesis/miners.go
View File

@ -78,7 +78,7 @@ func mkFakedSigSyscalls(base vm.SyscallBuilder) vm.SyscallBuilder {
}
// Note: Much of this is brittle, if the methodNum / param / return changes, it will break things
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, miners []genesis.Miner, nv network.Version) (cid.Cid, error) {
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.SyscallBuilder, sroot cid.Cid, miners []genesis.Miner, nv network.Version) (cid.Cid, error) {
cst := cbor.NewCborStore(cs.StateBlockstore())
av := actors.VersionForNetwork(nv)
@ -92,7 +92,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
Epoch: 0,
Rand: &fakeRand{},
Bstore: cs.StateBlockstore(),
Syscalls: mkFakedSigSyscalls(cs.VMSys()),
Syscalls: mkFakedSigSyscalls(sys),
CircSupplyCalc: csc,
NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
return nv

13
chain/messagepool/check.go
View File

@ -11,7 +11,6 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
)
@ -259,8 +258,14 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
Code: api.CheckStatusMessageValidity,
},
}
if err := m.ValidForBlockInclusion(0, build.NewestNetworkVersion); err != nil {
nv, err := mp.getNtwkVersion(epoch)
if err != nil {
check.OK = false
check.Err = fmt.Sprintf("error retrieving network version: %s", err.Error())
} else {
check.OK = true
}
if err := m.ValidForBlockInclusion(0, nv); err != nil {
check.OK = false
check.Err = fmt.Sprintf("syntactically invalid message: %s", err.Error())
} else {
@ -276,7 +281,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
// gas checks
// 4. Min Gas
minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
minGas := vm.PricelistByVersion(nv).OnChainMessage(m.ChainLength())
check = api.MessageCheckStatus{
Cid: m.Cid(),

66
chain/messagepool/messagepool.go
View File

@ -14,6 +14,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/network"
"github.com/hashicorp/go-multierror"
lru "github.com/hashicorp/golang-lru"
"github.com/ipfs/go-cid"
@ -29,6 +30,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
@ -147,6 +149,8 @@ type MessagePool struct {
minGasPrice types.BigInt
getNtwkVersion func(abi.ChainEpoch) (network.Version, error)
currentSize int
// pruneTrigger is a channel used to trigger a mempool pruning
@ -362,26 +366,28 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journ
if j == nil {
j = journal.NilJournal()
}
us := stmgr.DefaultUpgradeSchedule()
mp := &MessagePool{
ds: ds,
addSema: make(chan struct{}, 1),
closer: make(chan struct{}),
repubTk: build.Clock.Ticker(RepublishInterval),
repubTrigger: make(chan struct{}, 1),
localAddrs: make(map[address.Address]struct{}),
pending: make(map[address.Address]*msgSet),
keyCache: make(map[address.Address]address.Address),
minGasPrice: types.NewInt(0),
pruneTrigger: make(chan struct{}, 1),
pruneCooldown: make(chan struct{}, 1),
blsSigCache: cache,
sigValCache: verifcache,
changes: lps.New(50),
localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
api: api,
netName: netName,
cfg: cfg,
ds: ds,
addSema: make(chan struct{}, 1),
closer: make(chan struct{}),
repubTk: build.Clock.Ticker(RepublishInterval),
repubTrigger: make(chan struct{}, 1),
localAddrs: make(map[address.Address]struct{}),
pending: make(map[address.Address]*msgSet),
keyCache: make(map[address.Address]address.Address),
minGasPrice: types.NewInt(0),
getNtwkVersion: us.GetNtwkVersion,
pruneTrigger: make(chan struct{}, 1),
pruneCooldown: make(chan struct{}, 1),
blsSigCache: cache,
sigValCache: verifcache,
changes: lps.New(50),
localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
api: api,
netName: netName,
cfg: cfg,
evtTypes: [...]journal.EventType{
evtTypeMpoolAdd: j.RegisterEventType("mpool", "add"),
evtTypeMpoolRemove: j.RegisterEventType("mpool", "remove"),
@ -426,6 +432,27 @@ func New(api Provider, ds dtypes.MetadataDS, netName dtypes.NetworkName, j journ
return mp, nil
}
func (mp *MessagePool) ForEachPendingMessage(f func(cid.Cid) error) error {
mp.lk.Lock()
defer mp.lk.Unlock()
for _, mset := range mp.pending {
for _, m := range mset.msgs {
err := f(m.Cid())
if err != nil {
return err
}
err = f(m.Message.Cid())
if err != nil {
return err
}
}
}
return nil
}
func (mp *MessagePool) resolveToKey(ctx context.Context, addr address.Address) (address.Address, error) {
// check the cache
a, f := mp.keyCache[addr]
@ -589,8 +616,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
// For non local messages, if the message cannot be included in the next 20 blocks it returns
// a (soft) validation error.
func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
epoch := curTs.Height()
minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
minGas := vm.PricelistByVersion(build.NewestNetworkVersion).OnChainMessage(m.ChainLength())
if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {
return false, xerrors.Errorf("message will not be included in a block: %w", err)

3
chain/messagepool/messagepool_test.go
View File

@ -105,6 +105,7 @@ func (tma *testMpoolAPI) SubscribeHeadChanges(cb func(rev, app []*types.TipSet)
func (tma *testMpoolAPI) PutMessage(m types.ChainMsg) (cid.Cid, error) {
return cid.Undef, nil
}
func (tma *testMpoolAPI) IsLite() bool {
return false
}
@ -286,7 +287,7 @@ func TestCheckMessageBig(t *testing.T) {
From: from,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasLimit: 60000000,
GasFeeCap: types.NewInt(100),
GasPremium: types.NewInt(1),
Params: make([]byte, 41<<10), // 41KiB payload

2
chain/messagepool/selection.go
View File

@ -749,7 +749,7 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
}
curNonce++
minGas := vm.PricelistByEpoch(ts.Height()).OnChainMessage(m.ChainLength()).Total()
minGas := vm.PricelistByVersion(build.NewestNetworkVersion).OnChainMessage(m.ChainLength()).Total()
if m.Message.GasLimit < minGas {
break
}

57
chain/state/statetree.go
View File

@ -547,7 +547,7 @@ func (st *StateTree) Version() types.StateTreeVersion {
return st.version
}
func Diff(oldTree, newTree *StateTree) (map[string]types.Actor, error) {
func Diff(ctx context.Context, oldTree, newTree *StateTree) (map[string]types.Actor, error) {
out := map[string]types.Actor{}
var (
@ -555,33 +555,38 @@ func Diff(oldTree, newTree *StateTree) (map[string]types.Actor, error) {
buf = bytes.NewReader(nil)
)
if err := newTree.root.ForEach(&ncval, func(k string) error {
var act types.Actor
select {
case <-ctx.Done():
return ctx.Err()
default:
var act types.Actor
addr, err := address.NewFromBytes([]byte(k))
if err != nil {
return xerrors.Errorf("address in state tree was not valid: %w", err)
addr, err := address.NewFromBytes([]byte(k))
if err != nil {
return xerrors.Errorf("address in state tree was not valid: %w", err)
}
found, err := oldTree.root.Get(abi.AddrKey(addr), &ocval)
if err != nil {
return err
}
if found && bytes.Equal(ocval.Raw, ncval.Raw) {
return nil // not changed
}
buf.Reset(ncval.Raw)
err = act.UnmarshalCBOR(buf)
buf.Reset(nil)
if err != nil {
return err
}
out[addr.String()] = act
return nil
}
found, err := oldTree.root.Get(abi.AddrKey(addr), &ocval)
if err != nil {
return err
}
if found && bytes.Equal(ocval.Raw, ncval.Raw) {
return nil // not changed
}
buf.Reset(ncval.Raw)
err = act.UnmarshalCBOR(buf)
buf.Reset(nil)
if err != nil {
return err
}
out[addr.String()] = act
return nil
}); err != nil {
return nil, err
}

551
chain/stmgr/actors.go Normal file
View File

@ -0,0 +1,551 @@
package stmgr
import (
"bytes"
"context"
"os"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/network"
cid "github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/beacon"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
)
func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) {
state, err := sm.StateTree(st)
if err != nil {
return address.Undef, xerrors.Errorf("(get sset) failed to load state tree: %w", err)
}
act, err := state.GetActor(maddr)
if err != nil {
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return address.Undef, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
info, err := mas.Info()
if err != nil {
return address.Undef, xerrors.Errorf("failed to load actor info: %w", err)
}
return vm.ResolveToKeyAddr(state, sm.cs.ActorStore(ctx), info.Worker)
}
func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (power.Claim, power.Claim, bool, error) {
return GetPowerRaw(ctx, sm, ts.ParentState(), maddr)
}
func GetPowerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (power.Claim, power.Claim, bool, error) {
act, err := sm.LoadActorRaw(ctx, power.Address, st)
if err != nil {
return power.Claim{}, power.Claim{}, false, xerrors.Errorf("(get sset) failed to load power actor state: %w", err)
}
pas, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return power.Claim{}, power.Claim{}, false, err
}
tpow, err := pas.TotalPower()
if err != nil {
return power.Claim{}, power.Claim{}, false, err
}
var mpow power.Claim
var minpow bool
if maddr != address.Undef {
var found bool
mpow, found, err = pas.MinerPower(maddr)
if err != nil || !found {
return power.Claim{}, tpow, false, err
}
minpow, err = pas.MinerNominalPowerMeetsConsensusMinimum(maddr)
if err != nil {
return power.Claim{}, power.Claim{}, false, err
}
}
return mpow, tpow, minpow, nil
}
func PreCommitInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorPreCommitOnChainInfo, error) {
act, err := sm.LoadActor(ctx, maddr, ts)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
return mas.GetPrecommittedSector(sid)
}
func MinerSectorInfo(ctx context.Context, sm *StateManager, maddr address.Address, sid abi.SectorNumber, ts *types.TipSet) (*miner.SectorOnChainInfo, error) {
act, err := sm.LoadActor(ctx, maddr, ts)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
return mas.GetSector(sid)
}
func GetSectorsForWinningPoSt(ctx context.Context, nv network.Version, pv ffiwrapper.Verifier, sm *StateManager, st cid.Cid, maddr address.Address, rand abi.PoStRandomness) ([]builtin.SectorInfo, error) {
act, err := sm.LoadActorRaw(ctx, maddr, st)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
}
var provingSectors bitfield.BitField
if nv < network.Version7 {
allSectors, err := miner.AllPartSectors(mas, miner.Partition.AllSectors)
if err != nil {
return nil, xerrors.Errorf("get all sectors: %w", err)
}
faultySectors, err := miner.AllPartSectors(mas, miner.Partition.FaultySectors)
if err != nil {
return nil, xerrors.Errorf("get faulty sectors: %w", err)
}
provingSectors, err = bitfield.SubtractBitField(allSectors, faultySectors)
if err != nil {
return nil, xerrors.Errorf("calc proving sectors: %w", err)
}
} else {
provingSectors, err = miner.AllPartSectors(mas, miner.Partition.ActiveSectors)
if err != nil {
return nil, xerrors.Errorf("get active sectors sectors: %w", err)
}
}
numProvSect, err := provingSectors.Count()
if err != nil {
return nil, xerrors.Errorf("failed to count bits: %w", err)
}
// TODO(review): is this right? feels fishy to me
if numProvSect == 0 {
return nil, nil
}
info, err := mas.Info()
if err != nil {
return nil, xerrors.Errorf("getting miner info: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("getting miner ID: %w", err)
}
proofType, err := miner.WinningPoStProofTypeFromWindowPoStProofType(nv, info.WindowPoStProofType)
if err != nil {
return nil, xerrors.Errorf("determining winning post proof type: %w", err)
}
ids, err := pv.GenerateWinningPoStSectorChallenge(ctx, proofType, abi.ActorID(mid), rand, numProvSect)
if err != nil {
return nil, xerrors.Errorf("generating winning post challenges: %w", err)
}
iter, err := provingSectors.BitIterator()
if err != nil {
return nil, xerrors.Errorf("iterating over proving sectors: %w", err)
}
// Select winning sectors by _index_ in the all-sectors bitfield.
selectedSectors := bitfield.New()
prev := uint64(0)
for _, n := range ids {
sno, err := iter.Nth(n - prev)
if err != nil {
return nil, xerrors.Errorf("iterating over proving sectors: %w", err)
}
selectedSectors.Set(sno)
prev = n
}
sectors, err := mas.LoadSectors(&selectedSectors)
if err != nil {
return nil, xerrors.Errorf("loading proving sectors: %w", err)
}
out := make([]builtin.SectorInfo, len(sectors))
for i, sinfo := range sectors {
out[i] = builtin.SectorInfo{
SealProof: sinfo.SealProof,
SectorNumber: sinfo.SectorNumber,
SealedCID: sinfo.SealedCID,
}
}
return out, nil
}
func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (bool, error) {
act, err := sm.LoadActor(ctx, power.Address, ts)
if err != nil {
return false, xerrors.Errorf("failed to load power actor: %w", err)
}
spas, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return false, xerrors.Errorf("failed to load power actor state: %w", err)
}
_, ok, err := spas.MinerPower(maddr)
if err != nil {
return false, xerrors.Errorf("getting miner power: %w", err)
}
if !ok {
return true, nil
}
return false, nil
}
func GetStorageDeal(ctx context.Context, sm *StateManager, dealID abi.DealID, ts *types.TipSet) (*api.MarketDeal, error) {
act, err := sm.LoadActor(ctx, market.Address, ts)
if err != nil {
return nil, xerrors.Errorf("failed to load market actor: %w", err)
}
state, err := market.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load market actor state: %w", err)
}
proposals, err := state.Proposals()
if err != nil {
return nil, err
}
proposal, found, err := proposals.Get(dealID)
if err != nil {
return nil, err
} else if !found {
return nil, xerrors.Errorf(
"deal %d not found "+
"- deal may not have completed sealing before deal proposal "+
"start epoch, or deal may have been slashed",
dealID)
}
states, err := state.States()
if err != nil {
return nil, err
}
st, found, err := states.Get(dealID)
if err != nil {
return nil, err
}
if !found {
st = market.EmptyDealState()
}
return &api.MarketDeal{
Proposal: *proposal,
State: *st,
}, nil
}
func ListMinerActors(ctx context.Context, sm *StateManager, ts *types.TipSet) ([]address.Address, error) {
act, err := sm.LoadActor(ctx, power.Address, ts)
if err != nil {
return nil, xerrors.Errorf("failed to load power actor: %w", err)
}
powState, err := power.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load power actor state: %w", err)
}
return powState.ListAllMiners()
}
func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule, tsk types.TipSetKey, round abi.ChainEpoch, maddr address.Address, pv ffiwrapper.Verifier) (*api.MiningBaseInfo, error) {
ts, err := sm.ChainStore().LoadTipSet(tsk)
if err != nil {
return nil, xerrors.Errorf("failed to load tipset for mining base: %w", err)
}
prev, err := sm.ChainStore().GetLatestBeaconEntry(ts)
if err != nil {
if os.Getenv("LOTUS_IGNORE_DRAND") != "_yes_" {
return nil, xerrors.Errorf("failed to get latest beacon entry: %w", err)
}
prev = &types.BeaconEntry{}
}
entries, err := beacon.BeaconEntriesForBlock(ctx, bcs, round, ts.Height(), *prev)
if err != nil {
return nil, err
}
rbase := *prev
if len(entries) > 0 {
rbase = entries[len(entries)-1]
}
lbts, lbst, err := GetLookbackTipSetForRound(ctx, sm, ts, round)
if err != nil {
return nil, xerrors.Errorf("getting lookback miner actor state: %w", err)
}
act, err := sm.LoadActorRaw(ctx, maddr, lbst)
if xerrors.Is(err, types.ErrActorNotFound) {
_, err := sm.LoadActor(ctx, maddr, ts)
if err != nil {
return nil, xerrors.Errorf("loading miner in current state: %w", err)
}
return nil, nil
}
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor: %w", err)
}
mas, err := miner.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
}
buf := new(bytes.Buffer)
if err := maddr.MarshalCBOR(buf); err != nil {
return nil, xerrors.Errorf("failed to marshal miner address: %w", err)
}
prand, err := store.DrawRandomness(rbase.Data, crypto.DomainSeparationTag_WinningPoStChallengeSeed, round, buf.Bytes())
if err != nil {
return nil, xerrors.Errorf("failed to get randomness for winning post: %w", err)
}
nv := sm.GetNtwkVersion(ctx, ts.Height())
sectors, err := GetSectorsForWinningPoSt(ctx, nv, pv, sm, lbst, maddr, prand)
if err != nil {
return nil, xerrors.Errorf("getting winning post proving set: %w", err)
}
if len(sectors) == 0 {
return nil, nil
}
mpow, tpow, _, err := GetPowerRaw(ctx, sm, lbst, maddr)
if err != nil {
return nil, xerrors.Errorf("failed to get power: %w", err)
}
info, err := mas.Info()
if err != nil {
return nil, err
}
worker, err := sm.ResolveToKeyAddress(ctx, info.Worker, ts)
if err != nil {
return nil, xerrors.Errorf("resolving worker address: %w", err)
}
// TODO: Not ideal performance...This method reloads miner and power state (already looked up here and in GetPowerRaw)
eligible, err := MinerEligibleToMine(ctx, sm, maddr, ts, lbts)
if err != nil {
return nil, xerrors.Errorf("determining miner eligibility: %w", err)
}
return &api.MiningBaseInfo{
MinerPower: mpow.QualityAdjPower,
NetworkPower: tpow.QualityAdjPower,
Sectors: sectors,
WorkerKey: worker,
SectorSize: info.SectorSize,
PrevBeaconEntry: *prev,
BeaconEntries: entries,
EligibleForMining: eligible,
}, nil
}
func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) {
pact, err := sm.LoadActor(ctx, power.Address, ts)
if err != nil {
return false, xerrors.Errorf("loading power actor state: %w", err)
}
ps, err := power.Load(sm.cs.ActorStore(ctx), pact)
if err != nil {
return false, err
}
return ps.MinerNominalPowerMeetsConsensusMinimum(addr)
}
func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Address, baseTs *types.TipSet, lookbackTs *types.TipSet) (bool, error) {
hmp, err := minerHasMinPower(ctx, sm, addr, lookbackTs)
// TODO: We're blurring the lines between a "runtime network version" and a "Lotus upgrade epoch", is that unavoidable?
if sm.GetNtwkVersion(ctx, baseTs.Height()) <= network.Version3 {
return hmp, err
}
if err != nil {
return false, err
}
if !hmp {
return false, nil
}
// Post actors v2, also check MinerEligibleForElection with base ts
pact, err := sm.LoadActor(ctx, power.Address, baseTs)
if err != nil {
return false, xerrors.Errorf("loading power actor state: %w", err)
}
pstate, err := power.Load(sm.cs.ActorStore(ctx), pact)
if err != nil {
return false, err
}
mact, err := sm.LoadActor(ctx, addr, baseTs)
if err != nil {
return false, xerrors.Errorf("loading miner actor state: %w", err)
}
mstate, err := miner.Load(sm.cs.ActorStore(ctx), mact)
if err != nil {
return false, err
}
// Non-empty power claim.
if claim, found, err := pstate.MinerPower(addr); err != nil {
return false, err
} else if !found {
return false, err
} else if claim.QualityAdjPower.LessThanEqual(big.Zero()) {
return false, err
}
// No fee debt.
if debt, err := mstate.FeeDebt(); err != nil {
return false, err
} else if !debt.IsZero() {
return false, err
}
// No active consensus faults.
if mInfo, err := mstate.Info(); err != nil {
return false, err
} else if baseTs.Height() <= mInfo.ConsensusFaultElapsed {
return false, nil
}
return true, nil
}
func (sm *StateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) {
st, err := sm.ParentState(ts)
if err != nil {
return nil, nil, err
}
act, err := st.GetActor(addr)
if err != nil {
return nil, nil, err
}
actState, err := paych.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, nil, err
}
return act, actState, nil
}
func (sm *StateManager) GetMarketState(ctx context.Context, ts *types.TipSet) (market.State, error) {
st, err := sm.ParentState(ts)
if err != nil {
return nil, err
}
act, err := st.GetActor(market.Address)
if err != nil {
return nil, err
}
actState, err := market.Load(sm.cs.ActorStore(ctx), act)
if err != nil {
return nil, err
}
return actState, nil
}
func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (api.MarketBalance, error) {
mstate, err := sm.GetMarketState(ctx, ts)
if err != nil {
return api.MarketBalance{}, err
}
addr, err = sm.LookupID(ctx, addr, ts)
if err != nil {
return api.MarketBalance{}, err
}
var out api.MarketBalance
et, err := mstate.EscrowTable()
if err != nil {
return api.MarketBalance{}, err
}
out.Escrow, err = et.Get(addr)
if err != nil {
return api.MarketBalance{}, xerrors.Errorf("getting escrow balance: %w", err)
}
lt, err := mstate.LockedTable()
if err != nil {
return api.MarketBalance{}, err
}
out.Locked, err = lt.Get(addr)
if err != nil {
return api.MarketBalance{}, xerrors.Errorf("getting locked balance: %w", err)
}
return out, nil
}
var _ StateManagerAPI = (*StateManager)(nil)

4
chain/stmgr/call.go
View File

@ -64,7 +64,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
Epoch: pheight + 1,
Rand: store.NewChainRand(sm.cs, ts.Cids()),
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
Syscalls: sm.syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
BaseFee: types.NewInt(0),
@ -179,7 +179,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
Epoch: ts.Height() + 1,
Rand: r,
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.cs.VMSys(),
Syscalls: sm.syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
BaseFee: ts.Blocks()[0].ParentBaseFee,

326
chain/stmgr/execute.go Normal file
View File

@ -0,0 +1,326 @@
package stmgr
import (
"context"
"fmt"
"sync/atomic"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"go.opencensus.io/stats"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/cron"
"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/metrics"
)
func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEpoch, pstate cid.Cid, bms []store.BlockMessages, epoch abi.ChainEpoch, r vm.Rand, em ExecMonitor, baseFee abi.TokenAmount, ts *types.TipSet) (cid.Cid, cid.Cid, error) {
done := metrics.Timer(ctx, metrics.VMApplyBlocksTotal)
defer done()
partDone := metrics.Timer(ctx, metrics.VMApplyEarly)
defer func() {
partDone()
}()
makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) {
vmopt := &vm.VMOpts{
StateBase: base,
Epoch: epoch,
Rand: r,
Bstore: sm.cs.StateBlockstore(),
Syscalls: sm.syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NtwkVersion: sm.GetNtwkVersion,
BaseFee: baseFee,
LookbackState: LookbackStateGetterForTipset(sm, ts),
}
return sm.newVM(ctx, vmopt)
}
vmi, err := makeVmWithBaseState(pstate)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
}
runCron := func(epoch abi.ChainEpoch) error {
cronMsg := &types.Message{
To: cron.Address,
From: builtin.SystemActorAddr,
Nonce: uint64(epoch),
Value: types.NewInt(0),
GasFeeCap: types.NewInt(0),
GasPremium: types.NewInt(0),
GasLimit: build.BlockGasLimit * 10000, // Make super sure this is never too little
Method: cron.Methods.EpochTick,
Params: nil,
}
ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
if err != nil {
return err
}
if em != nil {
if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
return xerrors.Errorf("callback failed on cron message: %w", err)
}
}
if ret.ExitCode != 0 {
return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
}
return nil
}
for i := parentEpoch; i < epoch; i++ {
if i > parentEpoch {
// run cron for null rounds if any
if err := runCron(i); err != nil {
return cid.Undef, cid.Undef, err
}
pstate, err = vmi.Flush(ctx)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
}
}
// handle state forks
// XXX: The state tree
newState, err := sm.handleStateForks(ctx, pstate, i, em, ts)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
}
if pstate != newState {
vmi, err = makeVmWithBaseState(newState)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
}
}
vmi.SetBlockHeight(i + 1)
pstate = newState
}
partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyMessages)
var receipts []cbg.CBORMarshaler
processedMsgs := make(map[cid.Cid]struct{})
for _, b := range bms {
penalty := types.NewInt(0)
gasReward := big.Zero()
for _, cm := range append(b.BlsMessages, b.SecpkMessages...) {
m := cm.VMMessage()
if _, found := processedMsgs[m.Cid()]; found {
continue
}
r, err := vmi.ApplyMessage(ctx, cm)
if err != nil {
return cid.Undef, cid.Undef, err
}
receipts = append(receipts, &r.MessageReceipt)
gasReward = big.Add(gasReward, r.GasCosts.MinerTip)
penalty = big.Add(penalty, r.GasCosts.MinerPenalty)
if em != nil {
if err := em.MessageApplied(ctx, ts, cm.Cid(), m, r, false); err != nil {
return cid.Undef, cid.Undef, err
}
}
processedMsgs[m.Cid()] = struct{}{}
}
params, err := actors.SerializeParams(&reward.AwardBlockRewardParams{
Miner: b.Miner,
Penalty: penalty,
GasReward: gasReward,
WinCount: b.WinCount,
})
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to serialize award params: %w", err)
}
rwMsg := &types.Message{
From: builtin.SystemActorAddr,
To: reward.Address,
Nonce: uint64(epoch),
Value: types.NewInt(0),
GasFeeCap: types.NewInt(0),
GasPremium: types.NewInt(0),
GasLimit: 1 << 30,
Method: reward.Methods.AwardBlockReward,
Params: params,
}
ret, actErr := vmi.ApplyImplicitMessage(ctx, rwMsg)
if actErr != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to apply reward message for miner %s: %w", b.Miner, actErr)
}
if em != nil {
if err := em.MessageApplied(ctx, ts, rwMsg.Cid(), rwMsg, ret, true); err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("callback failed on reward message: %w", err)
}
}
if ret.ExitCode != 0 {
return cid.Undef, cid.Undef, xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
}
}
partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyCron)
if err := runCron(epoch); err != nil {
return cid.Cid{}, cid.Cid{}, err
}
partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyFlush)
rectarr := blockadt.MakeEmptyArray(sm.cs.ActorStore(ctx))
for i, receipt := range receipts {
if err := rectarr.Set(uint64(i), receipt); err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
}
}
rectroot, err := rectarr.Root()
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to build receipts amt: %w", err)
}
st, err := vmi.Flush(ctx)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("vm flush failed: %w", err)
}
stats.Record(ctx, metrics.VMSends.M(int64(atomic.LoadUint64(&vm.StatSends))),
metrics.VMApplied.M(int64(atomic.LoadUint64(&vm.StatApplied))))
return st, rectroot, nil
}
func (sm *StateManager) TipSetState(ctx context.Context, ts *types.TipSet) (st cid.Cid, rec cid.Cid, err error) {
ctx, span := trace.StartSpan(ctx, "tipSetState")
defer span.End()
if span.IsRecordingEvents() {
span.AddAttributes(trace.StringAttribute("tipset", fmt.Sprint(ts.Cids())))
}
ck := cidsToKey(ts.Cids())
sm.stlk.Lock()
cw, cwok := sm.compWait[ck]
if cwok {
sm.stlk.Unlock()
span.AddAttributes(trace.BoolAttribute("waited", true))
select {
case <-cw:
sm.stlk.Lock()
case <-ctx.Done():
return cid.Undef, cid.Undef, ctx.Err()
}
}
cached, ok := sm.stCache[ck]
if ok {
sm.stlk.Unlock()
span.AddAttributes(trace.BoolAttribute("cache", true))
return cached[0], cached[1], nil
}
ch := make(chan struct{})
sm.compWait[ck] = ch
defer func() {
sm.stlk.Lock()
delete(sm.compWait, ck)
if st != cid.Undef {
sm.stCache[ck] = []cid.Cid{st, rec}
}
sm.stlk.Unlock()
close(ch)
}()
sm.stlk.Unlock()
if ts.Height() == 0 {
// NB: This is here because the process that executes blocks requires that the
// block miner reference a valid miner in the state tree. Unless we create some
// magical genesis miner, this won't work properly, so we short circuit here
// This avoids the question of 'who gets paid the genesis block reward'
return ts.Blocks()[0].ParentStateRoot, ts.Blocks()[0].ParentMessageReceipts, nil
}
st, rec, err = sm.computeTipSetState(ctx, ts, sm.tsExecMonitor)
if err != nil {
return cid.Undef, cid.Undef, err
}
return st, rec, nil
}
func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, error) {
st, _, err := sm.computeTipSetState(ctx, ts, em)
return st, err
}
func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) {
var invocTrace []*api.InvocResult
st, err := sm.ExecutionTraceWithMonitor(ctx, ts, &InvocationTracer{trace: &invocTrace})
if err != nil {
return cid.Undef, nil, err
}
return st, invocTrace, nil
}
func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, cid.Cid, error) {
ctx, span := trace.StartSpan(ctx, "computeTipSetState")
defer span.End()
blks := ts.Blocks()
for i := 0; i < len(blks); i++ {
for j := i + 1; j < len(blks); j++ {
if blks[i].Miner == blks[j].Miner {
return cid.Undef, cid.Undef,
xerrors.Errorf("duplicate miner in a tipset (%s %s)",
blks[i].Miner, blks[j].Miner)
}
}
}
var parentEpoch abi.ChainEpoch
pstate := blks[0].ParentStateRoot
if blks[0].Height > 0 {
parent, err := sm.cs.GetBlock(blks[0].Parents[0])
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("getting parent block: %w", err)
}
parentEpoch = parent.Height
}
r := store.NewChainRand(sm.cs, ts.Cids())
blkmsgs, err := sm.cs.BlockMsgsForTipset(ts)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("getting block messages for tipset: %w", err)
}
baseFee := blks[0].ParentBaseFee
return sm.ApplyBlocks(ctx, parentEpoch, pstate, blkmsgs, blks[0].Height, r, em, baseFee, ts)
}

Some files were not shown because too many files have changed in this diff Show More