Compare commits
143
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39a0010c75 | ||
|
|
0bca36c126 | ||
|
|
f0df21b925 | ||
|
|
6ad8d72f2f | ||
|
|
6e6b1ec89d | ||
|
|
7b94399f9e | ||
|
|
cd7e9477b5 | ||
|
|
a0f1696f36 | ||
|
|
b418f82cf5 | ||
|
|
d6ad8517b7 | ||
|
|
221d1a0e3c | ||
|
|
dd0eec3994 | ||
|
|
4974dc65d5 | ||
|
|
21d078e7e1 | ||
|
|
f19f5269df | ||
|
|
a670359d78 | ||
|
|
28c550bbae | ||
|
|
7c858ece76 | ||
|
|
1cc59ade98 | ||
|
|
edcd2f34d4 | ||
|
|
5f49101566 | ||
|
|
25f61c8a5d | ||
|
|
12d1f8e636 | ||
|
|
270bad8144 | ||
|
|
e8e73e5374 | ||
|
|
8f5e6a864f | ||
|
|
07ffbf85e4 | ||
|
|
6a244ee8f0 | ||
|
|
583a3c6277 | ||
|
|
35895e897a | ||
|
|
970f3eff69 | ||
|
|
be24acbfbe | ||
|
|
da28416598 | ||
|
|
c06c8541f9 | ||
|
|
b3d02ea1f7 | ||
|
|
691da14996 | ||
|
|
bd3811e652 | ||
|
|
4076746141 | ||
|
|
f1d98361a6 | ||
|
|
7e02868ce2 | ||
|
|
0800de176e | ||
|
|
d1dc30b14d | ||
|
|
ae0680a5de | ||
|
|
3be2681824 | ||
|
|
b63858f210 | ||
|
|
932767d014 | ||
|
|
2e30891005 | ||
|
|
ab888c3a07 | ||
|
|
8cd305338a | ||
|
|
72391cca20 | ||
|
|
88efbfa6ff | ||
|
|
fcd630e529 | ||
|
|
073150a306 | ||
|
|
94318da0e6 | ||
|
|
9ba6a41ef4 | ||
|
|
5d047c59b1 | ||
|
|
80fcad5ad5 | ||
|
|
b6112c6fc9 | ||
|
|
650d6efb1a | ||
|
|
db6b94cf1e | ||
|
|
7c2341b43c | ||
|
|
19182999dd | ||
|
|
2a2b442aae | ||
|
|
d0353d1ef1 | ||
|
|
1250f8009f | ||
|
|
4b70d13a4b | ||
|
|
987fc0b77d | ||
|
|
6e9f480063 | ||
|
|
80b334d04f | ||
|
|
944aeed13e | ||
|
|
109a8242ae | ||
|
|
18f5997d02 | ||
|
|
160fab08aa | ||
|
|
9bdbbf33cb | ||
|
|
50e1dc6b6e | ||
|
|
48e8f54c6a | ||
|
|
776c216072 | ||
|
|
c60959ea9f | ||
|
|
62044c34b1 | ||
|
|
c8d328d386 | ||
|
|
fd879e0426 | ||
|
|
27f3170022 | ||
|
|
5b1241d213 | ||
|
|
eaee1578dd | ||
|
|
9e5c5b624b | ||
|
|
640d69f5cb | ||
|
|
36876c2601 | ||
|
|
39c1594505 | ||
|
|
86e1c854fa | ||
|
|
1af89236da | ||
|
|
9741867426 | ||
|
|
b7b98f94a6 | ||
|
|
fb50d68c32 | ||
|
|
692961e019 | ||
|
|
312cc94e6d | ||
|
|
ea71e4de60 | ||
|
|
4ed9cd152c | ||
|
|
0dbb1940f0 | ||
|
|
1452a361a9 | ||
|
|
503c08f999 | ||
|
|
6bee14d6fa | ||
|
|
839abdf910 | ||
|
|
6057f2e6d9 | ||
|
|
213dbc6bc8 | ||
|
|
fc77705229 | ||
|
|
12d7db4efb | ||
|
|
d7827da5bb | ||
|
|
b2ff2edd0c | ||
|
|
13e845ae51 | ||
|
|
0c90868929 | ||
|
|
44496afc66 | ||
|
|
8becd922cf | ||
|
|
16be5b5478 | ||
|
|
1e7095995b | ||
|
|
ffb995035f | ||
|
|
ffe7185443 | ||
|
|
44036d9a4a | ||
|
|
b6a7a8c987 | ||
|
|
905592a3dc | ||
|
|
9724248a49 | ||
|
|
ed058ae193 | ||
|
|
ca5a2eaa74 | ||
|
|
35b39063ac | ||
|
|
33c347956c | ||
|
|
a259ee307f | ||
|
|
46b78406b1 | ||
|
|
872843e491 | ||
|
|
a5ba556fff | ||
|
|
44d088be1e | ||
|
|
b40985a57c | ||
|
|
f8d32f5328 | ||
|
|
ba236fe258 | ||
|
|
2b5164c956 | ||
|
|
ca2d441b6d | ||
|
|
51696bb35d | ||
|
|
bed4c03a50 | ||
|
|
e041fe130b | ||
|
|
a07a0935fe | ||
|
|
de13b6edc5 | ||
|
|
303d5d6ff6 | ||
|
|
40b30d1fb1 | ||
|
|
dae6c757b5 | ||
|
|
31ecce3a72 |
+713
-977
File diff suppressed because it is too large
Load Diff
@@ -65,8 +65,6 @@ func main() {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// Redundantly flag both absolute and relative paths as excluded
|
||||
excluded[filepath.Join(repo, s)] = struct{}{}
|
||||
excluded[e] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -108,12 +106,10 @@ func main() {
|
||||
|
||||
// form the input data.
|
||||
type data struct {
|
||||
Networks []string
|
||||
ItestFiles []string
|
||||
UnitSuites map[string]string
|
||||
}
|
||||
in := data{
|
||||
Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"},
|
||||
ItestFiles: itests,
|
||||
UnitSuites: func() map[string]string {
|
||||
ret := make(map[string]string)
|
||||
|
||||
+698
-487
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
comment: off
|
||||
ignore:
|
||||
- "**/cbor_gen.go"
|
||||
- "api/test/**/*"
|
||||
- "api/test/*"
|
||||
- "gen/**/*"
|
||||
- "gen/*"
|
||||
- "cmd/lotus-shed/*"
|
||||
- "cmd/tvx/*"
|
||||
- "cmd/lotus-pcr/*"
|
||||
- "cmd/tvx/*"
|
||||
- "cmd/lotus-chainwatch/*"
|
||||
- "cmd/lotus-health/*"
|
||||
- "cmd/lotus-fountain/*"
|
||||
- "cmd/lotus-townhall/*"
|
||||
- "cmd/lotus-stats/*"
|
||||
- "cmd/lotus-pcr/*"
|
||||
github_checks:
|
||||
annotations: false
|
||||
@@ -1,2 +0,0 @@
|
||||
chain/actors/builtin/*/v* linguist-generated=true
|
||||
chain/actors/builtin/*/message* linguist-generated=true
|
||||
@@ -9,77 +9,84 @@ body:
|
||||
options:
|
||||
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
required: true
|
||||
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
|
||||
required: true
|
||||
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
|
||||
required: true
|
||||
- label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
|
||||
required: true
|
||||
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
|
||||
required: true
|
||||
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
|
||||
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
|
||||
required: true
|
||||
- label: I did not make any code changes to lotus.
|
||||
required: false
|
||||
- type: checkboxes
|
||||
- type: dropdown
|
||||
id: component-and-area
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a bug for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- lotus daemon - chain sync
|
||||
- lotus miner - mining and block production
|
||||
- lotus miner/worker - sealing
|
||||
- lotus miner - proving(WindowPoSt)
|
||||
- lotus miner/market - storage deal
|
||||
- lotus miner/market - retrieval deal
|
||||
- lotus client
|
||||
- lotus JSON-RPC API
|
||||
- lotus message management (mpool)
|
||||
- Other
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Lotus Version
|
||||
render: text
|
||||
description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
|
||||
placeholder: |
|
||||
e.g.
|
||||
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
|
||||
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
|
||||
Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0
|
||||
Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: ReproSteps
|
||||
attributes:
|
||||
label: Repro Steps
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
validations:
|
||||
required: false
|
||||
reuiqred: true
|
||||
- type: textarea
|
||||
id: Description
|
||||
attributes:
|
||||
label: Describe the Bug
|
||||
description: |
|
||||
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
|
||||
* What you were doing when you experienced the bug?
|
||||
* What you were doding when you experienced the bug?
|
||||
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
|
||||
* What is the expected behaviour?
|
||||
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
|
||||
* For proving issues, include the output of `lotus-miner proving` info.
|
||||
* For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
|
||||
render: bash
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extraInfo
|
||||
attributes:
|
||||
label: Logging Information
|
||||
render: text
|
||||
description: |
|
||||
Please provide debug logs of the problem, remember you can get set log level control for:
|
||||
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control).
|
||||
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
|
||||
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
|
||||
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem.
|
||||
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
|
||||
render: bash
|
||||
validations:
|
||||
required: true
|
||||
required: true
|
||||
- type: textarea
|
||||
id: RepoSteps
|
||||
attributes:
|
||||
label: Repo Steps
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
render: bash
|
||||
validations:
|
||||
required: false
|
||||
@@ -1,8 +0,0 @@
|
||||
blank_issues_enabled: true
|
||||
contact_links:
|
||||
- name: Ask a question about Lotus or get support
|
||||
url: https://github.com/filecoin-project/lotus/discussions/new/choose
|
||||
about: Ask a question or request support for using Lotus
|
||||
- name: Filecoin protocol feature or enhancement
|
||||
url: https://github.com/filecoin-project/FIPs/discussions/new/choose
|
||||
about: Write a discussion in the Filecoin Improvement Proposal repo
|
||||
@@ -7,41 +7,38 @@ body:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to create an improvement suggestion!
|
||||
options:
|
||||
- label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature.
|
||||
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
|
||||
required: true
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing an improvement request for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- type: textarea
|
||||
id: request
|
||||
attributes:
|
||||
label: Enhancement Suggestion
|
||||
description: A clear and concise description of the suggested enhancement?
|
||||
placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to...
|
||||
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
|
||||
required: true
|
||||
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
|
||||
required: true
|
||||
- label: I **have** a specific, actionable, and well motivated improvement to propose.
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: component
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are propoing improvement for
|
||||
options:
|
||||
- lotus daemon - chain sync
|
||||
- lotus miner - mining and block production
|
||||
- lotus miner/worker - sealing
|
||||
- lotus miner - proving(WindowPoSt)
|
||||
- lotus miner/market - storage deal
|
||||
- lotus miner/market - retrieval deal
|
||||
- lotus client
|
||||
- lotus JSON-RPC API
|
||||
- lotus message management (mpool)
|
||||
- Other
|
||||
- type: textarea
|
||||
id: request
|
||||
attributes:
|
||||
label: Use-Case
|
||||
description: How would this enhancement help you?
|
||||
placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable...
|
||||
label: Improvement Suggestion
|
||||
description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
|
||||
placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -7,29 +7,30 @@ body:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to create a new feature request!
|
||||
options:
|
||||
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
|
||||
required: true
|
||||
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
|
||||
required: true
|
||||
- label: I **have** a specific, actionable, and well motivated feature request to propose.
|
||||
required: true
|
||||
- type: checkboxes
|
||||
- type: dropdown
|
||||
id: component
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a new feature request for
|
||||
description: Please select the lotus component you are requesting a new feature for
|
||||
options:
|
||||
- label: lotus daemon - chain sync
|
||||
required: false
|
||||
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
|
||||
required: false
|
||||
- label: lotus miner/worker - sealing
|
||||
required: false
|
||||
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
|
||||
required: false
|
||||
- label: lotus JSON-RPC API
|
||||
required: false
|
||||
- label: lotus message management (mpool)
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- lotus daemon - chain sync
|
||||
- lotus miner - mining and block production
|
||||
- lotus miner/worker - sealing
|
||||
- lotus miner - proving(WindowPoSt)
|
||||
- lotus miner/market - storage deal
|
||||
- lotus miner/market - retrieval deal
|
||||
- lotus client
|
||||
- lotus JSON-RPC API
|
||||
- lotus message management (mpool)
|
||||
- Other
|
||||
- type: textarea
|
||||
id: request
|
||||
attributes:
|
||||
@@ -46,7 +47,7 @@ body:
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
id: alternates
|
||||
attributes:
|
||||
label: Describe alternatives you've considered
|
||||
description: A clear and concise description of any alternative solutions or features you've considered.
|
||||
@@ -59,3 +60,4 @@ body:
|
||||
description: Add any other context, design docs or screenshots about the feature request here.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
name: "M1 Bug Report For Deal Making"
|
||||
description: "File a bug report around deal making for the M1 releases"
|
||||
labels: [need/triage, kind/bug, M1-release]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to file a bug report!
|
||||
options:
|
||||
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
|
||||
required: true
|
||||
- label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
|
||||
required: true
|
||||
- label: I **am** reporting a bug around deal making. If not, create a [M1 Bug Report For Non Deal Making Issue](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_non_deal.yml).
|
||||
required: true
|
||||
- label: I have my log level set as instructed [here](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043678) and have logs available for troubleshooting.
|
||||
required: true
|
||||
- label: The deal is coming from one of the M1 clients(communitcated in the coordination slack channel).
|
||||
required: true
|
||||
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: lotus-componets
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Lotus Component
|
||||
description: Please select the lotus component you are filing a bug for
|
||||
options:
|
||||
- lotus miner market subsystem - storage deal
|
||||
- lotus miner market subsystem - retrieval deal
|
||||
- lotus miner - storage deal
|
||||
- lotus miner - retrieval deal
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Lotus Tag and Version
|
||||
description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`.
|
||||
validations:
|
||||
reuiqred: true
|
||||
- type: textarea
|
||||
id: Description
|
||||
attributes:
|
||||
label: Describe the Bug
|
||||
description: |
|
||||
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
|
||||
* What you were doding when you experienced the bug?
|
||||
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
|
||||
* What is the expected behaviour?
|
||||
render: bash
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: deal-status
|
||||
attributes:
|
||||
label: Deal Status
|
||||
description: What's the status of the deal?
|
||||
placeholder: |
|
||||
Please share the output of `lotus-miner storage-deals|retrieval-deals list [-v]` commands for the deal(s) in question.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: data-transfer-status
|
||||
attributes:
|
||||
label: Data Transfer Status
|
||||
description: What's the status of the data transfer?
|
||||
placeholder: |
|
||||
Please share the output of `lotus-miner data-transfers list -v` commands for the deal(s) in question.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logging
|
||||
attributes:
|
||||
label: Logging Information
|
||||
description: Please link to the whole of the miner logs on your side of the transaction. You can upload the logs to a [gist](https://gist.github.com).
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: RepoSteps
|
||||
attributes:
|
||||
label: Repo Steps (optional)
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
render: bash
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,81 @@
|
||||
name: "M1 Bug Report For Non Deal Making Issue"
|
||||
description: "File a bug report around non deal making issue for the M1 releases"
|
||||
labels: [need/triage, kind/bug, M1-release]
|
||||
body:
|
||||
- type: checkboxes
|
||||
id: checklist
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to file a bug report!
|
||||
options:
|
||||
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
|
||||
required: true
|
||||
- label: I **am** reporting a bug w.r.t one of the [M1 tags](https://github.com/filecoin-project/lotus/discussions/6852#discussioncomment-1043951). If not, choose another issue option [here](https://github.com/filecoin-project/lotus/issues/new/choose).
|
||||
required: true
|
||||
- label: I am **not** reporting a bug around deal making. If yes, create a [M1 Bug Report For Deal Making](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fbug%2CM1-release&template=m1_bug_report_deal.yml).
|
||||
required: true
|
||||
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: component-and-area
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a bug for
|
||||
options:
|
||||
- lotus daemon - chain sync **with** splitstore enabled
|
||||
- lotus daemon - chain sync **without** splitstore enabled
|
||||
- lotus miner - mining and block production
|
||||
- lotus miner/worker - sealing
|
||||
- lotus miner - proving(WindowPoSt)
|
||||
- lotus client
|
||||
- lotus JSON-RPC API
|
||||
- lotus message management (mpool)
|
||||
- Other
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Lotus Tag and Version
|
||||
description: Enter the lotus tag, output of `lotus version` and `lotus-miner version`.
|
||||
validations:
|
||||
reuiqred: true
|
||||
- type: textarea
|
||||
id: Description
|
||||
attributes:
|
||||
label: Describe the Bug
|
||||
description: |
|
||||
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
|
||||
* What you were doding when you experienced the bug?
|
||||
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
|
||||
* What is the expected behaviour?
|
||||
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
|
||||
* For proving issues, include the output of `lotus-miner proving` info.
|
||||
render: bash
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extraInfo
|
||||
attributes:
|
||||
label: Logging Information
|
||||
description: |
|
||||
Please provide debug logs of the problem, remember you can get set log level control for:
|
||||
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
|
||||
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
|
||||
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
|
||||
render: bash
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: RepoSteps
|
||||
attributes:
|
||||
label: Repo Steps
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
render: bash
|
||||
validations:
|
||||
required: false
|
||||
@@ -1,83 +0,0 @@
|
||||
name: "Bug Report - developer/service provider"
|
||||
description: "Bug report template about FEVM/FVM for developers/service providers"
|
||||
labels: [need/triage, kind/bug, area/fevm]
|
||||
body:
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Checklist
|
||||
description: Please check off the following boxes before continuing to file a bug report!
|
||||
options:
|
||||
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
|
||||
required: true
|
||||
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
|
||||
required: true
|
||||
- label: I did not make any code changes to lotus.
|
||||
required: false
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Lotus component
|
||||
description: Please select the lotus component you are filing a bug for
|
||||
options:
|
||||
- label: lotus Ethereum RPC
|
||||
required: false
|
||||
- label: lotus FVM - Lotus FVM interactions
|
||||
required: false
|
||||
- label: FEVM tooling
|
||||
required: false
|
||||
- label: Other
|
||||
required: false
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Lotus Version
|
||||
render: text
|
||||
description: Enter the output of `lotus version` if applicable.
|
||||
placeholder: |
|
||||
e.g.
|
||||
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
|
||||
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Repro Steps
|
||||
description: "Steps to reproduce the behavior"
|
||||
value: |
|
||||
1. Run '...'
|
||||
2. Do '...'
|
||||
3. See error '...'
|
||||
...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: Description
|
||||
attributes:
|
||||
label: Describe the Bug
|
||||
description: |
|
||||
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
|
||||
* What you were doing when you experienced the bug? What are you trying to build?
|
||||
* Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
|
||||
* What is the expected behaviour? Links to the actual code?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: toolingInfo
|
||||
attributes:
|
||||
label: Tooling
|
||||
render: text
|
||||
description: |
|
||||
What kind of tooling are you using:
|
||||
* Are you using ether.js, Alchemy, Hardhat, etc.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extraInfo
|
||||
attributes:
|
||||
label: Configuration Options
|
||||
render: text
|
||||
description: |
|
||||
Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM
|
||||
* lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section
|
||||
validations:
|
||||
required: true
|
||||
@@ -1,31 +0,0 @@
|
||||
---
|
||||
name: New Task
|
||||
about: A larger yet well-scoped task
|
||||
title: '<title>'
|
||||
labels: Needs Triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## User Story
|
||||
<!-- Why? -->
|
||||
|
||||
## Acceptance Criteria
|
||||
<!-- What? -->
|
||||
<!-- add description-->
|
||||
|
||||
```[tasklist]
|
||||
### Deliverables
|
||||
|
||||
```
|
||||
|
||||
## Technical Breakdown
|
||||
```[tasklist]
|
||||
### Development
|
||||
|
||||
```
|
||||
|
||||
```[tasklist]
|
||||
### Testing
|
||||
|
||||
```
|
||||
@@ -1,24 +0,0 @@
|
||||
## Related Issues
|
||||
<!-- Link issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made -->
|
||||
|
||||
## Proposed Changes
|
||||
<!-- A clear list of the changes being made -->
|
||||
|
||||
## Additional Info
|
||||
<!-- Callouts, links to documentation, and etc -->
|
||||
|
||||
## Checklist
|
||||
|
||||
Before you mark the PR ready for review, please make sure that:
|
||||
|
||||
- [ ] Commits have a clear commit message.
|
||||
- [ ] PR title is in the form of of `<PR type>: <area>: <change being made>`
|
||||
- example: ` fix: mempool: Introduce a cache for valid signatures`
|
||||
- `PR type`: fix, feat, build, chore, ci, docs, perf, refactor, revert, style, test
|
||||
- `area`, e.g. api, chain, state, market, mempool, multisig, networking, paych, proving, sealing, wallet, deps
|
||||
- [ ] If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section.
|
||||
- [ ] New features have usage guidelines and / or documentation updates in
|
||||
- [ ] [Lotus Documentation](https://lotus.filecoin.io)
|
||||
- [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
|
||||
- [ ] Tests exist for new functionality or change in behavior
|
||||
- [ ] CI is green
|
||||
@@ -13,14 +13,10 @@ name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- 'release/*'
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches:
|
||||
- master
|
||||
- 'release/*'
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
@@ -37,17 +33,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-go@v3
|
||||
- uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: '1.18.8'
|
||||
go-version: '1.16.4'
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: go
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
@@ -56,7 +52,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
@@ -70,4 +66,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
uses: github/codeql-action/analyze@v1
|
||||
|
||||
@@ -29,5 +29,5 @@ jobs:
|
||||
days-before-pr-close: 2
|
||||
remove-stale-when-updated: true
|
||||
enable-statistics: true
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
name: sync-master-main
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: update remote branch main
|
||||
run: |
|
||||
# overrides the remote branch (origin:github) `main`
|
||||
git push origin --force master:main
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: testground run
|
||||
uses: testground/testground-github-action@v1
|
||||
uses: coryschwartz/testground-github-action@v1.1
|
||||
with:
|
||||
backend_addr: ${{ matrix.backend_addr }}
|
||||
backend_proto: ${{ matrix.backend_proto }}
|
||||
|
||||
+6
-13
@@ -1,3 +1,6 @@
|
||||
/AppDir
|
||||
/appimage-builder-cache
|
||||
*.AppImage
|
||||
/lotus
|
||||
/lotus-miner
|
||||
/lotus-worker
|
||||
@@ -6,6 +9,7 @@
|
||||
/lotus-chainwatch
|
||||
/lotus-shed
|
||||
/lotus-sim
|
||||
/lotus-pond
|
||||
/lotus-townhall
|
||||
/lotus-fountain
|
||||
/lotus-stats
|
||||
@@ -17,6 +21,8 @@
|
||||
/docgen-md
|
||||
/docgen-openrpc
|
||||
/bench.json
|
||||
/lotuspond/front/node_modules
|
||||
/lotuspond/front/build
|
||||
/cmd/lotus-townhall/townhall/node_modules
|
||||
/cmd/lotus-townhall/townhall/build
|
||||
/cmd/lotus-townhall/townhall/package-lock.json
|
||||
@@ -34,23 +40,10 @@ build/paramfetch.sh
|
||||
/bundle
|
||||
/darwin
|
||||
/linux
|
||||
*.snap
|
||||
|
||||
*-fuzz.zip
|
||||
/chain/types/work_msg/
|
||||
bin/ipget
|
||||
bin/tmp/*
|
||||
.idea
|
||||
.vscode
|
||||
scratchpad
|
||||
|
||||
build/builtin-actors/v*
|
||||
build/builtin-actors/*.car
|
||||
|
||||
dist/
|
||||
|
||||
|
||||
# The following files are checked into git and result
|
||||
# in dirty git state if removed from the docker context
|
||||
!extern/filecoin-ffi/rust/filecoin.pc
|
||||
!extern/test-vectors
|
||||
|
||||
+6
-6
@@ -12,9 +12,9 @@ linters:
|
||||
- unconvert
|
||||
- staticcheck
|
||||
- varcheck
|
||||
- structcheck
|
||||
- deadcode
|
||||
- scopelint
|
||||
- unused
|
||||
|
||||
# We don't want to skip builtin/
|
||||
skip-dirs-use-default: false
|
||||
@@ -25,7 +25,7 @@ skip-dirs:
|
||||
|
||||
issues:
|
||||
exclude:
|
||||
- "by other packages, and that stutters; consider calling this"
|
||||
- "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this"
|
||||
- "Potential file inclusion via variable"
|
||||
- "should have( a package)? comment"
|
||||
- "Error return value of `logging.SetLogLevel` is not checked"
|
||||
@@ -37,12 +37,12 @@ issues:
|
||||
- "string .* has .* occurrences, make it a constant"
|
||||
- "a blank import should be only in a main or test package, or have a comment justifying it"
|
||||
- "package comment should be of the form"
|
||||
- "Potential hardcoded credentials"
|
||||
- "Use of weak random number generator"
|
||||
- "xerrors.* is deprecated"
|
||||
|
||||
exclude-use-default: false
|
||||
exclude-rules:
|
||||
- path: lotuspond
|
||||
linters:
|
||||
- errcheck
|
||||
|
||||
- path: node/modules/lp2p
|
||||
linters:
|
||||
@@ -51,7 +51,7 @@ issues:
|
||||
- path: build/params_.*\.go
|
||||
linters:
|
||||
- golint
|
||||
|
||||
|
||||
- path: api/apistruct/struct.go
|
||||
linters:
|
||||
- golint
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
project_name: lotus
|
||||
|
||||
universal_binaries:
|
||||
- id: lotus
|
||||
replace: true
|
||||
name_template: lotus
|
||||
- id: lotus-miner
|
||||
replace: true
|
||||
name_template: lotus-miner
|
||||
- id: lotus-worker
|
||||
replace: true
|
||||
name_template: lotus-worker
|
||||
|
||||
builds:
|
||||
- id: lotus
|
||||
binary: lotus
|
||||
builder: prebuilt
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
goamd64:
|
||||
- v1
|
||||
ignore:
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
prebuilt:
|
||||
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus
|
||||
- id: lotus-miner
|
||||
binary: lotus-miner
|
||||
builder: prebuilt
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
goamd64:
|
||||
- v1
|
||||
ignore:
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
prebuilt:
|
||||
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner
|
||||
- id: lotus-worker
|
||||
binary: lotus-worker
|
||||
builder: prebuilt
|
||||
goos:
|
||||
- darwin
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
goamd64:
|
||||
- v1
|
||||
ignore:
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
prebuilt:
|
||||
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker
|
||||
|
||||
archives:
|
||||
- id: primary
|
||||
format: tar.gz
|
||||
wrap_in_directory: true
|
||||
name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
files:
|
||||
# this is a dumb but required hack so it doesn't include the default files
|
||||
# https://github.com/goreleaser/goreleaser/issues/602
|
||||
- _n_o_n_e_*
|
||||
|
||||
release:
|
||||
github:
|
||||
owner: filecoin-project
|
||||
name: lotus
|
||||
prerelease: auto
|
||||
name_template: "v{{.Version}}"
|
||||
|
||||
brews:
|
||||
- tap:
|
||||
owner: filecoin-project
|
||||
name: homebrew-lotus
|
||||
branch: master
|
||||
ids:
|
||||
- primary
|
||||
install: |
|
||||
bin.install "lotus"
|
||||
bin.install "lotus-miner"
|
||||
bin.install "lotus-worker"
|
||||
test: |
|
||||
system "#{bin}/lotus --version"
|
||||
system "#{bin}/lotus-miner --version"
|
||||
system "#{bin}/lotus-worker --version"
|
||||
folder: Formula
|
||||
homepage: "https://filecoin.io"
|
||||
description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
|
||||
license: MIT
|
||||
skip_upload: auto
|
||||
dependencies:
|
||||
- name: hwloc
|
||||
|
||||
# produced manually so we can include cid checksums
|
||||
checksum:
|
||||
disable: true
|
||||
|
||||
snapshot:
|
||||
name_template: "{{ .Version }}"
|
||||
@@ -0,0 +1 @@
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0" y="0" viewBox="0 0 127 127" xml:space="preserve" enable-background="new 0 0 127 127"><style type="text/css">.st0{fill:#00d2d6}.st1{fill:#fff}</style><g><path class="st0" d="M63.5,127C28.5,127.1-0.2,98.4,0,63.2C0.2,28.3,28.6-0.2,63.9,0c34.8,0.2,63.3,28.7,63.1,64 C126.7,98.7,98.5,127.1,63.5,127z M71.4,57.6c5.5,0.8,11,1.5,16.5,2.3c0.5-1.7,0.9-3.1,1.3-4.7c-5.7-0.8-11.2-1.7-17.1-2.5 c2-7,3.7-13.7,5.8-20.2c0.7-2.2,2.3-4.2,3.9-5.9c2.1-2.2,5-1.7,6.8,0.7c0.7,1,1.4,2.1,2.3,2.9c1.1,1.1,2.8,1.6,4,0.6 c0.8-0.7,0.7-2.4,0.8-3.6c0-0.5-0.6-1.1-1-1.6c-2-2.3-4.7-3.1-7.5-3.2c-6.3-0.3-10.9,3-14.5,7.8c-3.5,4.8-5.1,10.5-6.8,16.2 c-0.5,1.6-0.9,3.3-1.4,5.1c-6.2-0.9-12.1-1.7-18.2-2.6c-0.2,1.6-0.4,3.2-0.6,4.8c6,0.9,11.8,1.8,17.8,2.7c-0.8,3.4-1.5,6.5-2.3,9.7 c-5.8-0.8-11.4-1.6-17-2.4c-0.2,1.8-0.4,3.2-0.6,4.8c5.6,0.9,11,1.7,16.5,2.5c0,0.6,0.1,1,0,1.3c-1.7,7.4-3.4,14.8-5.3,22.2 c-0.9,3.5-2.4,6.9-5.3,9.3c-2.4,2-5,1.7-6.8-0.8c-0.8-1.1-1.5-2.5-2.6-3.3c-0.8-0.6-2.5-0.9-3.1-0.5c-0.9,0.7-1.5,2.2-1.4,3.3 c0.1,1,1,2.2,1.9,2.8c3,2.3,6.5,2.6,10,1.9c5.9-1.2,10.1-4.9,12.7-10.1c2-4.1,3.6-8.5,5-12.9c1.3-4,2.2-8,3.3-12.2 c5.8,0.8,11.5,1.7,17.3,2.5c0.5-1.7,0.9-3.2,1.4-4.8c-6.1-0.9-11.9-1.7-17.7-2.6C70.1,64,70.7,60.9,71.4,57.6z"/><path class="st1" d="M71.4,57.6c-0.7,3.3-1.3,6.4-2,9.8c5.9,0.9,11.7,1.7,17.7,2.6c-0.5,1.6-0.9,3.1-1.4,4.8 c-5.8-0.8-11.5-1.7-17.3-2.5c-1.1,4.2-2,8.3-3.3,12.2c-1.4,4.4-3,8.7-5,12.9c-2.6,5.2-6.8,8.9-12.7,10.1c-3.5,0.7-7,0.4-10-1.9 c-0.9-0.7-1.8-1.8-1.9-2.8c-0.1-1.1,0.5-2.7,1.4-3.3c0.6-0.5,2.3-0.1,3.1,0.5c1.1,0.8,1.8,2.1,2.6,3.3c1.8,2.5,4.4,2.9,6.8,0.8 c2.9-2.5,4.4-5.8,5.3-9.3c1.9-7.3,3.6-14.8,5.3-22.2c0.1-0.3,0-0.7,0-1.3c-5.4-0.8-10.8-1.7-16.5-2.5c0.2-1.6,0.4-3,0.6-4.8 c5.6,0.8,11.1,1.6,17,2.4c0.8-3.2,1.5-6.4,2.3-9.7c-6-0.9-11.7-1.8-17.8-2.7c0.2-1.6,0.4-3.2,0.6-4.8c6.1,0.9,12,1.7,18.2,2.6 c0.5-1.8,0.9-3.5,1.4-5.1c1.7-5.6,3.2-11.3,6.8-16.2c3.6-4.9,8.1-8.1,14.5-7.8c2.8,0.1,5.5,0.9,7.5,3.2c0.4,0.5,1,1.1,1,1.6 c-0.1,1.2,0,2.9-0.8,3.6c-1.1,1.1-2.8,0.5-4-0.6c-0.9-0.9-1.6-1.9-2.3-2.9c-1.8-2.4-4.7-2.9-6.8-0.7c-1.6,1.7-3.2,3.7-3.9,5.9 C75.7,39.4,74,46,72,53c5.9,0.9,11.4,1.7,17.1,2.5c-0.5,1.6-0.9,3.1-1.3,4.7C82.3,59.1,76.9,58.3,71.4,57.6z"/></g></svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
@@ -0,0 +1,73 @@
|
||||
version: 1
|
||||
AppDir:
|
||||
path: ./AppDir
|
||||
app_info:
|
||||
id: io.filecoin.lotus
|
||||
name: Lotus
|
||||
icon: icon
|
||||
version: latest
|
||||
exec: usr/bin/lotus
|
||||
exec_args: $@
|
||||
apt:
|
||||
arch: amd64
|
||||
allow_unauthenticated: true
|
||||
sources:
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal main restricted
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal universe
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates universe
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal multiverse
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates multiverse
|
||||
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
|
||||
universe multiverse
|
||||
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security main restricted
|
||||
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security universe
|
||||
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security multiverse
|
||||
- sourceline: deb https://cli-assets.heroku.com/apt ./
|
||||
- sourceline: deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu focal main
|
||||
- sourceline: deb http://ppa.launchpad.net/git-core/ppa/ubuntu focal main
|
||||
- sourceline: deb http://archive.canonical.com/ubuntu focal partner
|
||||
include:
|
||||
- ocl-icd-libopencl1
|
||||
- libhwloc15
|
||||
exclude: []
|
||||
files:
|
||||
include:
|
||||
- /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
|
||||
- /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
|
||||
- /usr/lib/x86_64-linux-gnu/libm-2.31.so
|
||||
- /usr/lib/x86_64-linux-gnu/libdl-2.31.so
|
||||
- /usr/lib/x86_64-linux-gnu/libc-2.31.so
|
||||
- /usr/lib/x86_64-linux-gnu/libudev.so.1.6.17
|
||||
exclude:
|
||||
- usr/share/man
|
||||
- usr/share/doc/*/README.*
|
||||
- usr/share/doc/*/changelog.*
|
||||
- usr/share/doc/*/NEWS.*
|
||||
- usr/share/doc/*/TODO.*
|
||||
test:
|
||||
fedora:
|
||||
image: appimagecrafters/tests-env:fedora-30
|
||||
command: ./AppRun
|
||||
use_host_x: true
|
||||
debian:
|
||||
image: appimagecrafters/tests-env:debian-stable
|
||||
command: ./AppRun
|
||||
use_host_x: true
|
||||
arch:
|
||||
image: appimagecrafters/tests-env:archlinux-latest
|
||||
command: ./AppRun
|
||||
use_host_x: true
|
||||
centos:
|
||||
image: appimagecrafters/tests-env:centos-7
|
||||
command: ./AppRun
|
||||
use_host_x: true
|
||||
ubuntu:
|
||||
image: appimagecrafters/tests-env:ubuntu-xenial
|
||||
command: ./AppRun
|
||||
use_host_x: true
|
||||
AppImage:
|
||||
arch: x86_64
|
||||
update-information: guess
|
||||
sign-key: None
|
||||
|
||||
+184
-3243
File diff suppressed because it is too large
Load Diff
-136
@@ -1,136 +0,0 @@
|
||||
#####################################
|
||||
FROM golang:1.20.7-bullseye AS lotus-builder
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
|
||||
|
||||
ENV XDG_CACHE_HOME="/tmp"
|
||||
|
||||
### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
|
||||
ENV RUSTUP_HOME=/usr/local/rustup \
|
||||
CARGO_HOME=/usr/local/cargo \
|
||||
PATH=/usr/local/cargo/bin:$PATH \
|
||||
RUST_VERSION=1.63.0
|
||||
|
||||
RUN set -eux; \
|
||||
dpkgArch="$(dpkg --print-architecture)"; \
|
||||
case "${dpkgArch##*-}" in \
|
||||
amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
|
||||
arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
|
||||
*) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
|
||||
esac; \
|
||||
url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
|
||||
wget "$url"; \
|
||||
echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
|
||||
chmod +x rustup-init; \
|
||||
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
|
||||
rm rustup-init; \
|
||||
chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
|
||||
rustup --version; \
|
||||
cargo --version; \
|
||||
rustc --version;
|
||||
|
||||
COPY ./ /opt/filecoin
|
||||
WORKDIR /opt/filecoin
|
||||
|
||||
RUN scripts/docker-git-state-check.sh
|
||||
|
||||
### make configurable filecoin-ffi build
|
||||
ARG FFI_BUILD_FROM_SOURCE=0
|
||||
ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
|
||||
|
||||
RUN make clean deps
|
||||
|
||||
ARG RUSTFLAGS=""
|
||||
ARG GOFLAGS=""
|
||||
|
||||
RUN make buildall
|
||||
|
||||
#####################################
|
||||
FROM ubuntu:20.04 AS lotus-base
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
# Base resources
|
||||
COPY --from=lotus-builder /etc/ssl/certs /etc/ssl/certs
|
||||
COPY --from=lotus-builder /lib/*/libdl.so.2 /lib/
|
||||
COPY --from=lotus-builder /lib/*/librt.so.1 /lib/
|
||||
COPY --from=lotus-builder /lib/*/libgcc_s.so.1 /lib/
|
||||
COPY --from=lotus-builder /lib/*/libutil.so.1 /lib/
|
||||
COPY --from=lotus-builder /usr/lib/*/libltdl.so.7 /lib/
|
||||
COPY --from=lotus-builder /usr/lib/*/libnuma.so.1 /lib/
|
||||
COPY --from=lotus-builder /usr/lib/*/libhwloc.so.* /lib/
|
||||
COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
|
||||
|
||||
RUN useradd -r -u 532 -U fc \
|
||||
&& mkdir -p /etc/OpenCL/vendors \
|
||||
&& echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
|
||||
|
||||
#####################################
|
||||
FROM lotus-base AS lotus
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY scripts/docker-lotus-entrypoint.sh /
|
||||
|
||||
ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
|
||||
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV DOCKER_LOTUS_IMPORT_WALLET ""
|
||||
|
||||
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
|
||||
ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
#####################################
|
||||
FROM lotus-base AS lotus-all-in-one
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
|
||||
ENV WALLET_PATH /var/lib/lotus-wallet
|
||||
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-seed /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/
|
||||
COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
|
||||
|
||||
RUN mkdir /var/tmp/filecoin-proof-parameters
|
||||
RUN mkdir /var/lib/lotus
|
||||
RUN mkdir /var/lib/lotus-miner
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus
|
||||
RUN chown fc: /var/lib/lotus-miner
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
|
||||
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/lib/lotus-worker
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
|
||||
EXPOSE 1234
|
||||
EXPOSE 2345
|
||||
EXPOSE 3456
|
||||
EXPOSE 1777
|
||||
@@ -0,0 +1,228 @@
|
||||
FROM golang:1.16.4 AS builder-deps
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
|
||||
|
||||
ARG RUST_VERSION=nightly
|
||||
ENV XDG_CACHE_HOME="/tmp"
|
||||
|
||||
ENV RUSTUP_HOME=/usr/local/rustup \
|
||||
CARGO_HOME=/usr/local/cargo \
|
||||
PATH=/usr/local/cargo/bin:$PATH
|
||||
|
||||
RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
|
||||
chmod +x rustup-init; \
|
||||
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
|
||||
rm rustup-init; \
|
||||
chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
|
||||
rustup --version; \
|
||||
cargo --version; \
|
||||
rustc --version;
|
||||
|
||||
|
||||
FROM builder-deps AS builder-local
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY ./ /opt/filecoin
|
||||
WORKDIR /opt/filecoin
|
||||
RUN make clean deps
|
||||
|
||||
|
||||
FROM builder-local AS builder
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
WORKDIR /opt/filecoin
|
||||
|
||||
ARG RUSTFLAGS=""
|
||||
ARG GOFLAGS=""
|
||||
|
||||
RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway
|
||||
|
||||
|
||||
FROM ubuntu:20.04 AS base
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
# Base resources
|
||||
COPY --from=builder /etc/ssl/certs /etc/ssl/certs
|
||||
COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/
|
||||
COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1 /lib/
|
||||
COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/
|
||||
COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1 /lib/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7 /lib/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
|
||||
|
||||
RUN useradd -r -u 532 -U fc
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY scripts/docker-lotus-entrypoint.sh /
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
|
||||
ENV DOCKER_LOTUS_IMPORT_WALLET ""
|
||||
|
||||
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
|
||||
ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
###
|
||||
FROM base AS lotus-wallet
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
|
||||
ENV WALLET_PATH /var/lib/lotus-wallet
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1777
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-wallet"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
###
|
||||
FROM base AS lotus-gateway
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 1234
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-gateway"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus-miner
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY scripts/docker-lotus-miner-entrypoint.sh /
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV DOCKER_LOTUS_MINER_INIT true
|
||||
|
||||
RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
|
||||
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 2345
|
||||
|
||||
ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
FROM base AS lotus-worker
|
||||
MAINTAINER Lotus Development Team
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
|
||||
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
|
||||
VOLUME /var/lib/lotus-worker
|
||||
|
||||
USER fc
|
||||
|
||||
EXPOSE 3456
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/lotus-worker"]
|
||||
|
||||
CMD ["-help"]
|
||||
|
||||
|
||||
###
|
||||
from base as lotus-all-in-one
|
||||
|
||||
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
|
||||
ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http
|
||||
ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1
|
||||
ENV LOTUS_JAEGER_AGENT_PORT 6831
|
||||
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
|
||||
ENV LOTUS_PATH /var/lib/lotus
|
||||
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
|
||||
ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http
|
||||
ENV WALLET_PATH /var/lib/lotus-wallet
|
||||
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
|
||||
ENV DOCKER_LOTUS_MINER_INIT true
|
||||
|
||||
COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
|
||||
COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
|
||||
|
||||
RUN mkdir /var/tmp/filecoin-proof-parameters
|
||||
RUN mkdir /var/lib/lotus
|
||||
RUN mkdir /var/lib/lotus-miner
|
||||
RUN mkdir /var/lib/lotus-worker
|
||||
RUN mkdir /var/lib/lotus-wallet
|
||||
RUN chown fc: /var/tmp/filecoin-proof-parameters
|
||||
RUN chown fc: /var/lib/lotus
|
||||
RUN chown fc: /var/lib/lotus-miner
|
||||
RUN chown fc: /var/lib/lotus-worker
|
||||
RUN chown fc: /var/lib/lotus-wallet
|
||||
|
||||
|
||||
VOLUME /var/tmp/filecoin-proof-parameters
|
||||
VOLUME /var/lib/lotus
|
||||
VOLUME /var/lib/lotus-miner
|
||||
VOLUME /var/lib/lotus-worker
|
||||
VOLUME /var/lib/lotus-wallet
|
||||
|
||||
EXPOSE 1234
|
||||
EXPOSE 2345
|
||||
EXPOSE 3456
|
||||
EXPOSE 1777
|
||||
@@ -1 +0,0 @@
|
||||
1.20.7
|
||||
@@ -1,142 +0,0 @@
|
||||
|
||||
<!-- TOC -->
|
||||
|
||||
- [`lotus` Release Flow](#lotus-release-flow)
|
||||
- [Purpose](#purpose)
|
||||
- [High-level Summary](#high-level-summary)
|
||||
- [Motivation and Requirements](#motivation-and-requirements)
|
||||
- [Adopted Conventions](#adopted-conventions)
|
||||
- [Major Releases](#major-releases)
|
||||
- [Mandatory Releases](#mandatory-releases)
|
||||
- [Feature Releases](#feature-releases)
|
||||
- [Examples Scenarios](#examples-scenarios)
|
||||
- [Release Cycle](#release-cycle)
|
||||
- [Patch Releases](#patch-releases)
|
||||
- [Performing a Release](#performing-a-release)
|
||||
- [Security Fix Policy](#security-fix-policy)
|
||||
- [FAQ](#faq)
|
||||
- [Why aren't Go major versions used more?](#why-arent-go-major-versions-used-more)
|
||||
- [Related Items](#related-items)
|
||||
|
||||
<!-- /TOC -->
|
||||
|
||||
# `lotus` Release Flow
|
||||
|
||||
## Purpose
|
||||
|
||||
This document aims to describe how the Lotus team plans to ship releases of the Lotus implementation of Filecoin network. Interested parties can expect new releases to come out as described in this document.
|
||||
|
||||
## High-level Summary
|
||||
|
||||
- **Major releases** (1.0.0, 2.0.0, etc.) are reserved for significant changes to the Filecoin protocol that transform the network and its usecases. Such changes could include the addition of new Virtual Machines to the protocol, major upgrades to the Proofs of Replication used by Filecoin, etc.
|
||||
- Even minor releases (1.2.0, 1.4.0, etc.) of the Lotus software correspond to **mandatory releases** as they ship Filecoin network upgrades. Users **must** upgrade to these releases before a certain time in order to keep in sync with the Filecoin network. We aim to ensure there is at least one week before the upgrade deadline.
|
||||
- Patch versions of even minor releases (1.2.1, 1.4.2, etc.) correspond to **hotfix releases**. Such releases will only be shipped when a critical fix is needed to be applied on top of a mandatory release.
|
||||
- Odd minor releases (1.3.0, 1.5.0, etc.), as well as patch releases in these series (1.3.1, 1.5.2, etc.) correspond to **feature releases** with new development and bugfixes. These releases are not mandatory, but still highly recommended, **as they may contain critical security fixes**.
|
||||
- We aim to ship a new feature release of the Lotus software every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule.
|
||||
|
||||
## Motivation and Requirements
|
||||
|
||||
Our primary motivation is for users of the Lotus software (storage providers, storage clients, developers, etc.) to have a clear idea about when they can expect Lotus releases, and what they can expect in a release.
|
||||
|
||||
In order to achieve this, we need the following from our release process and conventions:
|
||||
|
||||
- Lotus version conventions make it immediately obvious whether a new Lotus release is mandatory or not. A release is mandatory if it ships a network upgrade to the Filecoin protocol.
|
||||
- The ability to ship critical fixes on top of mandatory releases, so as to avoid forcing users to consume larger unrelated changes.
|
||||
- A regular cadence of feature releases, so that users can know when a new Lotus release will be available for consumption.
|
||||
- The ability to ship any number of feature releases between two mandatory releases.
|
||||
- A clear description of the various stages of testing that a Lotus Release Candidate (RC) goes through.
|
||||
- Lotus Release issues will present a single source of truth for what may be contained in Lotus releases, including security fixes, and how they will be disclosed.
|
||||
|
||||
## Adopted Conventions
|
||||
|
||||
This section describes the conventions we have adopted. Users of Lotus are encouraged to review this in detail so as to be informed when new Lotus releases are shipped.
|
||||
|
||||
### Major Releases
|
||||
|
||||
Bumps to the Lotus major version number (1.0.0, 2.0.0, etc.) are reserved for significant changes to the Filecoin protocol that dramatically transform the network itself. Such changes could include the addition of new Virtual Machines to the protocol, major upgrades to the Proofs of Replication used by Filecoin, etc. These releases are expected to take lots of time to develop and will be rare. See also "Why aren't Go major versions used more?" below.
|
||||
|
||||
### Mandatory Releases
|
||||
|
||||
Even bumps to the Lotus minor version number (1.2.0, 1.4.0, etc.) are reserved for **mandatory releases** that ship Filecoin network upgrades. Users **must** upgrade to these releases before a certain time in order to keep in sync with the Filecoin network.
|
||||
|
||||
Depending on the scope of the upgrade, these releases may take up to several weeks to fully develop and test. We aim to ensure there are at least 2 weeks between the publication of the final Lotus release and the Filecoin network upgrade deadline.
|
||||
|
||||
These releases do not follow a regular cadence, as they are developed in lockstep with the other implementations of the Filecoin protocol. As of August 2021, the developers aim to ship 3-4 Filecoin network upgrades a year, though smaller security-critical upgrades may occur unpredictably.
|
||||
|
||||
Mandatory releases are somewhat sensitive since all Lotus users are forced to upgrade to them at the same time. As a result, they will be shipped on top of the most recent stable release of Lotus, which will generally be the latest Lotus release that has been in production for more than 2 weeks. (Note: given this rule, the basis of a mandatory release could be a mandatory release or a feature release depending on timing). Mandatory releases will not include any new feature development or bugfixes that haven't already baked in production for 2+ weeks, except for the changes needed for the network upgrade itself. Further, any critical fixes that are needed after the network upgrade will be shipped as patch version bumps to the mandatory release (1.2.1, 1.2.2, etc.) This prevents users from being forced to quickly digest unnecessary changes.
|
||||
|
||||
Users should generally aim to always upgrade to a new even minor version release since they either introduce a mandatory network upgrade or a critical fix.
|
||||
|
||||
### Feature Releases
|
||||
|
||||
All releases under an odd minor version number indicate **feature releases**. These could include releases such as 1.3.0, 1.3.1, 1.5.2, etc.
|
||||
|
||||
Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes. There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
|
||||
|
||||
We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the [Release Cycle section](#release-cycle).
|
||||
|
||||
### Examples Scenarios
|
||||
|
||||
- **Scenario 1**: **Lotus 1.12.0 shipped a network upgrade, and no network upgrades are needed for a long while.**
|
||||
|
||||
In this case, the next feature release will be Lotus 1.13.0. In three-week intervals, we will ship Lotus 1.13.1, 1.13.2, and 1.13.3, all containing new features and bug fixes.
|
||||
|
||||
Let us assume that after the release of 1.13.3, a critical issue is discovered and a hotfix quickly developed. This hotfix will then be shipped in **both** 1.12.1 and 1.13.4. Users who have already upgrade to the 1.13 series can simply upgrade to 1.13.4. Users who have chosen to still be on 1.12.0, however, can use 1.12.1 to patch the critical issue without being forced to consume all the changes in the 1.13 series.
|
||||
|
||||
- **Scenario 2**: **Lotus 1.12.0 shipped a network upgrade, but the need for an unexpected network upgrade soon arises**
|
||||
|
||||
In this case, the Lotus 1.13 series will be dropped entirely, including any RCs that may have been undergoing testing. Instead, the network upgrade will be shipped as Lotus 1.14.0, built on top of Lotus 1.12.0. It will thus include no unnecessary changes, only the work needed to support the new network upgrade.
|
||||
|
||||
Any changes that were being worked on in the 1.13.0 series will then get applied on top of Lotus 1.14.0 and get shipped as Lotus 1.15.0.
|
||||
|
||||
## Release Cycle
|
||||
|
||||
A mandatory release process should take about 3-6 weeks, depending on the amount and the overall complexity of new features being introduced to the network protocol. It may also be shorter if there is a network incident that requires an emergency upgrade. A feature release process should take about 2-3 weeks.
|
||||
|
||||
The start time of the mandatory release process is subject to the network upgrade timeline. We will start a new feature release process every 3 weeks on Tuesdays, regardless of when the previous release landed unless it's still ongoing.
|
||||
|
||||
### Patch Releases
|
||||
|
||||
**Mandatory Release**
|
||||
|
||||
If we encounter a serious bug in a mandatory release post a network upgrade, we will create a patch release based on this release. Strictly only the fix to the bug will be included in the patch, and the bug fix will be backported to the master (dev) branch, and any ongoing feature release branch if applicable.
|
||||
|
||||
Patch release process for the mandatory releases will follow a compressed release cycle from hours to days depending on the severity and the impact to the network of the bug. In a patch release:
|
||||
|
||||
1. Automated and internal testing (stage 0 and 1) will be compressed into a few hours.
|
||||
2. Stage 2-3 will be skipped or shortened case by case.
|
||||
|
||||
Some patch releases, especially ones fixing one or more complex bugs that doesn't require a follow-up mandatory upgrade, may undergo the full release process.
|
||||
|
||||
**Feature Release**
|
||||
|
||||
Patch releases in odd minor releases (1.3.0, 1.5.0, etc.) like 1.3.1, 1.5.2 and etc are corresponding to another **feature releases** with new development and bugfixes. These releases are not mandatory, but still **highly** recommended, **as they may contain critical security fixes**.
|
||||
|
||||
---
|
||||
|
||||
### Performing a Release
|
||||
|
||||
At the beginning of each release cycle, we will generate our "Release tracking issue", which is populated with the content at [https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md)
|
||||
|
||||
This template will be used to track major goals we have, a planned shipping date, and a complete release checklist tied to a specific release.
|
||||
|
||||
### Security Fix Policy
|
||||
|
||||
Any release may contain security fixes. Unless the fix addresses a bug being exploited in the wild, the fix will *not* be called out in the release notes. Please make sure to update ASAP.
|
||||
|
||||
By policy, the team will usually wait until about 3 weeks after the final release to announce any fixed security issues. However, depending on the impact and ease of discovery of the issue, the team may wait more or less time.
|
||||
|
||||
It is important to always update to the latest version ASAP and file issues if you're unable to update for some reason.
|
||||
|
||||
Finally, unless a security issue is actively being exploited or a significant number of users are unable to update to the latest version (e.g., due to a difficult migration, breaking changes, etc.), security fixes will *not* be backported to previous releases.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Why aren't Go major versions used more?
|
||||
|
||||
Golang tightly couples source code with versioning (major versions beyond v1 leak into import paths). This poses logistical difficulties to using major versions here. Concretely, if we were to pick a policy that bumped the major version on every network upgrade, we would disrupt every single downstream library/application that consumed the native Lotus API (e.g., libraries depending on the JSON-RPC client, testground tests). They would need to update their code every single time that we released a network breaking change, even if it brought on zero expectation of breakage for the Golang APIs that they depend on. In this scenario, we are signaling breakage on the wrong API surface! We're signaling breakage on the Go level, when what breaks is the network protocol.
|
||||
|
||||
## Related Items
|
||||
|
||||
1. [Release Issue template](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md)
|
||||
2. [Lotus Release Flow Discussion](https://github.com/filecoin-project/lotus/discussions/7053): Leave a comment if you have any questions or feedbacks with regard to the lotus release flow.
|
||||
@@ -5,14 +5,10 @@ all: build
|
||||
|
||||
unexport GOFLAGS
|
||||
|
||||
GOCC?=go
|
||||
|
||||
GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
|
||||
GOVERSIONMIN:=$(shell cat GO_VERSION_MIN | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
|
||||
|
||||
ifeq ($(shell expr $(GOVERSION) \< $(GOVERSIONMIN)), 1)
|
||||
GOVERSION:=$(shell go version | cut -d' ' -f 3 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
|
||||
ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
|
||||
$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
|
||||
$(error Update Golang to version to at least $(shell cat GO_VERSION_MIN))
|
||||
$(error Update Golang to version to at least 1.16.0)
|
||||
endif
|
||||
|
||||
# git modules that need to be loaded
|
||||
@@ -66,7 +62,7 @@ CLEAN+=build/.update-modules
|
||||
deps: $(BUILD_DEPS)
|
||||
.PHONY: deps
|
||||
|
||||
build-devnets: build lotus-seed lotus-shed
|
||||
build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
|
||||
.PHONY: build-devnets
|
||||
|
||||
debug: GOFLAGS+=-tags=debug
|
||||
@@ -78,6 +74,9 @@ debug: build-devnets
|
||||
calibnet: GOFLAGS+=-tags=calibnet
|
||||
calibnet: build-devnets
|
||||
|
||||
nerpanet: GOFLAGS+=-tags=nerpanet
|
||||
nerpanet: build-devnets
|
||||
|
||||
butterflynet: GOFLAGS+=-tags=butterflynet
|
||||
butterflynet: build-devnets
|
||||
|
||||
@@ -86,32 +85,32 @@ interopnet: build-devnets
|
||||
|
||||
lotus: $(BUILD_DEPS)
|
||||
rm -f lotus
|
||||
$(GOCC) build $(GOFLAGS) -o lotus ./cmd/lotus
|
||||
go build $(GOFLAGS) -o lotus ./cmd/lotus
|
||||
|
||||
.PHONY: lotus
|
||||
BINS+=lotus
|
||||
|
||||
lotus-miner: $(BUILD_DEPS)
|
||||
rm -f lotus-miner
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
|
||||
go build $(GOFLAGS) -o lotus-miner ./cmd/lotus-miner
|
||||
.PHONY: lotus-miner
|
||||
BINS+=lotus-miner
|
||||
|
||||
lotus-worker: $(BUILD_DEPS)
|
||||
rm -f lotus-worker
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
|
||||
go build $(GOFLAGS) -o lotus-worker ./cmd/lotus-seal-worker
|
||||
.PHONY: lotus-worker
|
||||
BINS+=lotus-worker
|
||||
|
||||
lotus-shed: $(BUILD_DEPS)
|
||||
rm -f lotus-shed
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
|
||||
go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
|
||||
.PHONY: lotus-shed
|
||||
BINS+=lotus-shed
|
||||
|
||||
lotus-gateway: $(BUILD_DEPS)
|
||||
rm -f lotus-gateway
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
|
||||
go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
|
||||
.PHONY: lotus-gateway
|
||||
BINS+=lotus-gateway
|
||||
|
||||
@@ -135,91 +134,112 @@ install-worker:
|
||||
install-app:
|
||||
install -C ./$(APP) /usr/local/bin/$(APP)
|
||||
|
||||
uninstall: uninstall-daemon uninstall-miner uninstall-worker
|
||||
.PHONY: uninstall
|
||||
|
||||
uninstall-daemon:
|
||||
rm -f /usr/local/bin/lotus
|
||||
|
||||
uninstall-miner:
|
||||
rm -f /usr/local/bin/lotus-miner
|
||||
|
||||
uninstall-worker:
|
||||
rm -f /usr/local/bin/lotus-worker
|
||||
|
||||
# TOOLS
|
||||
|
||||
lotus-seed: $(BUILD_DEPS)
|
||||
rm -f lotus-seed
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
|
||||
go build $(GOFLAGS) -o lotus-seed ./cmd/lotus-seed
|
||||
|
||||
.PHONY: lotus-seed
|
||||
BINS+=lotus-seed
|
||||
|
||||
benchmarks:
|
||||
$(GOCC) run github.com/whyrusleeping/bencher ./... > bench.json
|
||||
go run github.com/whyrusleeping/bencher ./... > bench.json
|
||||
@echo Submitting results
|
||||
@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
|
||||
.PHONY: benchmarks
|
||||
|
||||
lotus-pond: 2k
|
||||
go build -o lotus-pond ./lotuspond
|
||||
.PHONY: lotus-pond
|
||||
BINS+=lotus-pond
|
||||
|
||||
lotus-pond-front:
|
||||
(cd lotuspond/front && npm i && CI=false npm run build)
|
||||
.PHONY: lotus-pond-front
|
||||
|
||||
lotus-pond-app: lotus-pond-front lotus-pond
|
||||
.PHONY: lotus-pond-app
|
||||
|
||||
lotus-townhall:
|
||||
rm -f lotus-townhall
|
||||
go build -o lotus-townhall ./cmd/lotus-townhall
|
||||
.PHONY: lotus-townhall
|
||||
BINS+=lotus-townhall
|
||||
|
||||
lotus-townhall-front:
|
||||
(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
|
||||
.PHONY: lotus-townhall-front
|
||||
|
||||
lotus-townhall-app: lotus-touch lotus-townhall-front
|
||||
.PHONY: lotus-townhall-app
|
||||
|
||||
lotus-fountain:
|
||||
rm -f lotus-fountain
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
|
||||
$(GOCC) run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
|
||||
go build -o lotus-fountain ./cmd/lotus-fountain
|
||||
.PHONY: lotus-fountain
|
||||
BINS+=lotus-fountain
|
||||
|
||||
lotus-chainwatch:
|
||||
rm -f lotus-chainwatch
|
||||
go build $(GOFLAGS) -o lotus-chainwatch ./cmd/lotus-chainwatch
|
||||
.PHONY: lotus-chainwatch
|
||||
BINS+=lotus-chainwatch
|
||||
|
||||
lotus-bench:
|
||||
rm -f lotus-bench
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-bench ./cmd/lotus-bench
|
||||
go build -o lotus-bench ./cmd/lotus-bench
|
||||
.PHONY: lotus-bench
|
||||
BINS+=lotus-bench
|
||||
|
||||
lotus-stats:
|
||||
rm -f lotus-stats
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
|
||||
go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
|
||||
.PHONY: lotus-stats
|
||||
BINS+=lotus-stats
|
||||
|
||||
lotus-pcr:
|
||||
rm -f lotus-pcr
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
|
||||
go build $(GOFLAGS) -o lotus-pcr ./cmd/lotus-pcr
|
||||
.PHONY: lotus-pcr
|
||||
BINS+=lotus-pcr
|
||||
|
||||
lotus-health:
|
||||
rm -f lotus-health
|
||||
$(GOCC) build -o lotus-health ./cmd/lotus-health
|
||||
go build -o lotus-health ./cmd/lotus-health
|
||||
.PHONY: lotus-health
|
||||
BINS+=lotus-health
|
||||
|
||||
lotus-wallet: $(BUILD_DEPS)
|
||||
lotus-wallet:
|
||||
rm -f lotus-wallet
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-wallet ./cmd/lotus-wallet
|
||||
go build -o lotus-wallet ./cmd/lotus-wallet
|
||||
.PHONY: lotus-wallet
|
||||
BINS+=lotus-wallet
|
||||
|
||||
lotus-keygen:
|
||||
rm -f lotus-keygen
|
||||
$(GOCC) build -o lotus-keygen ./cmd/lotus-keygen
|
||||
go build -o lotus-keygen ./cmd/lotus-keygen
|
||||
.PHONY: lotus-keygen
|
||||
BINS+=lotus-keygen
|
||||
|
||||
testground:
|
||||
$(GOCC) build -tags testground -o /dev/null ./cmd/lotus
|
||||
go build -tags testground -o /dev/null ./cmd/lotus
|
||||
.PHONY: testground
|
||||
BINS+=testground
|
||||
|
||||
|
||||
tvx:
|
||||
rm -f tvx
|
||||
$(GOCC) build -o tvx ./cmd/tvx
|
||||
go build -o tvx ./cmd/tvx
|
||||
.PHONY: tvx
|
||||
BINS+=tvx
|
||||
|
||||
install-chainwatch: lotus-chainwatch
|
||||
install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
|
||||
|
||||
lotus-sim: $(BUILD_DEPS)
|
||||
rm -f lotus-sim
|
||||
$(GOCC) build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
|
||||
go build $(GOFLAGS) -o lotus-sim ./cmd/lotus-sim
|
||||
.PHONY: lotus-sim
|
||||
BINS+=lotus-sim
|
||||
|
||||
@@ -241,13 +261,21 @@ install-miner-service: install-miner install-daemon-service
|
||||
@echo
|
||||
@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
|
||||
|
||||
install-chainwatch-service: install-chainwatch install-daemon-service
|
||||
mkdir -p /etc/systemd/system
|
||||
mkdir -p /var/log/lotus
|
||||
install -C -m 0644 ./scripts/lotus-chainwatch.service /etc/systemd/system/lotus-chainwatch.service
|
||||
systemctl daemon-reload
|
||||
@echo
|
||||
@echo "chainwatch service installed. Don't forget to run 'sudo systemctl start lotus-chainwatch' to start it and 'sudo systemctl enable lotus-chainwatch' for it to be enabled on startup."
|
||||
|
||||
install-main-services: install-miner-service
|
||||
|
||||
install-all-services: install-main-services
|
||||
install-all-services: install-main-services install-chainwatch-service
|
||||
|
||||
install-services: install-main-services
|
||||
|
||||
clean-daemon-service: clean-miner-service
|
||||
clean-daemon-service: clean-miner-service clean-chainwatch-service
|
||||
-systemctl stop lotus-daemon
|
||||
-systemctl disable lotus-daemon
|
||||
rm -f /etc/systemd/system/lotus-daemon.service
|
||||
@@ -259,6 +287,12 @@ clean-miner-service:
|
||||
rm -f /etc/systemd/system/lotus-miner.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-chainwatch-service:
|
||||
-systemctl stop lotus-chainwatch
|
||||
-systemctl disable lotus-chainwatch
|
||||
rm -f /etc/systemd/system/lotus-chainwatch.service
|
||||
systemctl daemon-reload
|
||||
|
||||
clean-main-services: clean-daemon-service
|
||||
|
||||
clean-all-services: clean-main-services
|
||||
@@ -285,32 +319,25 @@ dist-clean:
|
||||
.PHONY: dist-clean
|
||||
|
||||
type-gen: api-gen
|
||||
$(GOCC) run ./gen/main.go
|
||||
$(GOCC) generate -x ./...
|
||||
go run ./gen/main.go
|
||||
go generate -x ./...
|
||||
goimports -w api/
|
||||
|
||||
actors-code-gen:
|
||||
$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
|
||||
$(GOCC) run ./chain/actors/agen
|
||||
$(GOCC) fmt ./...
|
||||
|
||||
actors-gen: actors-code-gen fiximports
|
||||
.PHONY: actors-gen
|
||||
|
||||
bundle-gen:
|
||||
$(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES)
|
||||
$(GOCC) fmt ./build/...
|
||||
.PHONY: bundle-gen
|
||||
method-gen: api-gen
|
||||
(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
|
||||
|
||||
actors-gen:
|
||||
go run ./chain/actors/agen
|
||||
go fmt ./...
|
||||
|
||||
api-gen:
|
||||
$(GOCC) run ./gen/api
|
||||
go run ./gen/api
|
||||
goimports -w api
|
||||
goimports -w api
|
||||
.PHONY: api-gen
|
||||
|
||||
cfgdoc-gen:
|
||||
$(GOCC) run ./node/config/cfgdocgen > ./node/config/doc_gen.go
|
||||
go run ./node/config/cfgdocgen > ./node/config/doc_gen.go
|
||||
|
||||
appimage: lotus
|
||||
rm -rf appimage-builder-cache || true
|
||||
@@ -321,12 +348,12 @@ appimage: lotus
|
||||
cp ./lotus AppDir/usr/bin/
|
||||
appimage-builder
|
||||
|
||||
docsgen: docsgen-md docsgen-openrpc fiximports
|
||||
docsgen: docsgen-md docsgen-openrpc
|
||||
|
||||
docsgen-md-bin: api-gen actors-gen
|
||||
$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
|
||||
go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
|
||||
docsgen-openrpc-bin: api-gen actors-gen
|
||||
$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
|
||||
go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
|
||||
|
||||
docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
|
||||
|
||||
@@ -338,7 +365,7 @@ docsgen-md-storage: docsgen-md-bin
|
||||
docsgen-md-worker: docsgen-md-bin
|
||||
./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
|
||||
|
||||
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
|
||||
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
|
||||
|
||||
docsgen-openrpc-full: docsgen-openrpc-bin
|
||||
./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
|
||||
@@ -346,33 +373,24 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
|
||||
./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
|
||||
docsgen-openrpc-worker: docsgen-openrpc-bin
|
||||
./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
|
||||
docsgen-openrpc-gateway: docsgen-openrpc-bin
|
||||
./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
|
||||
|
||||
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
|
||||
|
||||
fiximports:
|
||||
./scripts/fiximports
|
||||
|
||||
gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
|
||||
@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'"
|
||||
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
|
||||
@echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli"
|
||||
.PHONY: gen
|
||||
|
||||
jen: gen
|
||||
|
||||
snap: lotus lotus-miner lotus-worker
|
||||
snapcraft
|
||||
# snapcraft upload ./lotus_*.snap
|
||||
|
||||
# separate from gen because it needs binaries
|
||||
docsgen-cli: lotus lotus-miner lotus-worker
|
||||
python3 ./scripts/generate-lotus-cli.py
|
||||
./lotus config default > documentation/en/default-lotus-config.toml
|
||||
./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
|
||||
python ./scripts/generate-lotus-cli.py
|
||||
.PHONY: docsgen-cli
|
||||
|
||||
print-%:
|
||||
@echo $*=$($*)
|
||||
|
||||
circleci:
|
||||
go generate -x ./.circleci
|
||||
go generate -x ./.circleci
|
||||
@@ -1,5 +1,5 @@
|
||||
<p align="center">
|
||||
<a href="https://lotus.filecoin.io/" title="Filecoin Docs">
|
||||
<a href="https://docs.filecoin.io/" title="Filecoin Docs">
|
||||
<img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
|
||||
</a>
|
||||
</p>
|
||||
@@ -10,7 +10,7 @@
|
||||
<a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
|
||||
<a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
|
||||
<a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.18.8-blue.svg" /></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
|
||||
<br>
|
||||
</p>
|
||||
|
||||
@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
|
||||
|
||||
> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
|
||||
|
||||
For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
|
||||
For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
@@ -31,7 +31,7 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
|
||||
These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
|
||||
|
||||
- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
|
||||
- [builtin-actors](https://github.com/filecoin-project/builtin-actors)
|
||||
- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
|
||||
|
||||
## Contribute
|
||||
|
||||
@@ -64,17 +64,17 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
|
||||
|
||||
Fedora:
|
||||
```
|
||||
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
|
||||
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
|
||||
```
|
||||
|
||||
For other distributions you can find the required dependencies [here.](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms) For instructions specific to macOS, you can find them [here.](https://lotus.filecoin.io/lotus/install/macos/)
|
||||
For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
|
||||
|
||||
#### Go
|
||||
|
||||
To build Lotus, you need a working installation of [Go 1.20.10 or higher](https://golang.org/dl/):
|
||||
To build Lotus, you need a working installation of [Go 1.16.4 or higher](https://golang.org/dl/):
|
||||
|
||||
```bash
|
||||
wget -c https://golang.org/dl/go1.20.10.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
|
||||
wget -c https://golang.org/dl/go1.16.4.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
|
||||
```
|
||||
|
||||
**TIP:**
|
||||
@@ -101,7 +101,7 @@ Note: The default branch `master` is the dev branch where the latest new feature
|
||||
|
||||
2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
|
||||
|
||||
If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://lotus.filecoin.io/lotus/manage/switch-networks/) before proceeding.
|
||||
If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
|
||||
|
||||
For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
|
||||
|
||||
@@ -113,8 +113,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
|
||||
|
||||
Currently, the latest code on the _master_ branch corresponds to mainnet.
|
||||
|
||||
3. If you are in China, see "[Lotus: tips when running in China](https://lotus.filecoin.io/lotus/configure/nodes-in-china/)".
|
||||
4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://lotus.filecoin.io/lotus/install/prerequisites/). Note, if you are building the proof binaries from source, [installing rustup](https://lotus.filecoin.io/lotus/install/linux/#rustup) is also needed.
|
||||
3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
|
||||
4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
|
||||
|
||||
5. Build and install Lotus:
|
||||
|
||||
@@ -123,17 +123,16 @@ Note: The default branch `master` is the dev branch where the latest new feature
|
||||
|
||||
# Or to join a testnet or devnet:
|
||||
make clean calibnet # Calibration with min 32GiB sectors
|
||||
make clean nerpanet # Nerpa with min 512MiB sectors
|
||||
|
||||
sudo make install
|
||||
```
|
||||
|
||||
This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
|
||||
|
||||
`lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://lotus.filecoin.io/lotus/configure/defaults/#environment-variables) for information on how to customize the Lotus folder.
|
||||
`lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
|
||||
|
||||
6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
|
||||
|
||||
7. (Optional) Follow the [Setting Up Prometheus and Grafana](https://github.com/filecoin-project/lotus/tree/master/metrics/README.md) guide for detailed instructions on setting up a working monitoring system running against a local running lotus node.
|
||||
6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+2
-11
@@ -3,14 +3,12 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/journal/alerting"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -35,10 +33,6 @@ type Common interface {
|
||||
LogList(context.Context) ([]string, error) //perm:write
|
||||
LogSetLevel(context.Context, string, string) error //perm:write
|
||||
|
||||
// LogAlerts returns list of all, active and inactive alerts tracked by the
|
||||
// node
|
||||
LogAlerts(ctx context.Context) ([]alerting.Alert, error) //perm:admin
|
||||
|
||||
// MethodGroup: Common
|
||||
|
||||
// Version provides information about API provider
|
||||
@@ -50,9 +44,6 @@ type Common interface {
|
||||
// trigger graceful shutdown
|
||||
Shutdown(context.Context) error //perm:admin
|
||||
|
||||
// StartTime returns node start time
|
||||
StartTime(context.Context) (time.Time, error) //perm:read
|
||||
|
||||
// Session returns a random UUID of api provider session
|
||||
Session(context.Context) (uuid.UUID, error) //perm:read
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
)
|
||||
|
||||
const (
|
||||
EOutOfGas = iota + jsonrpc.FirstUserCode
|
||||
EActorNotFound
|
||||
)
|
||||
|
||||
type ErrOutOfGas struct{}
|
||||
|
||||
func (e *ErrOutOfGas) Error() string {
|
||||
return "call ran out of gas"
|
||||
}
|
||||
|
||||
type ErrActorNotFound struct{}
|
||||
|
||||
func (e *ErrActorNotFound) Error() string {
|
||||
return "actor not found"
|
||||
}
|
||||
|
||||
var RPCErrors = jsonrpc.NewErrors()
|
||||
|
||||
func ErrorIsIn(err error, errorTypes []error) bool {
|
||||
for _, etype := range errorTypes {
|
||||
tmp := reflect.New(reflect.PointerTo(reflect.ValueOf(etype).Elem().Type())).Interface()
|
||||
if errors.As(err, tmp) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func init() {
|
||||
RPCErrors.Register(EOutOfGas, new(*ErrOutOfGas))
|
||||
RPCErrors.Register(EActorNotFound, new(*ErrActorNotFound))
|
||||
}
|
||||
+61
-311
@@ -6,34 +6,29 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v9/market"
|
||||
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
abinetwork "github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/types/ethtypes"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=mocks/mock_full.go -package=mocks . FullNode
|
||||
@@ -42,7 +37,6 @@ import (
|
||||
type ChainIO interface {
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
ChainPutObj(context.Context, blocks.Block) error
|
||||
}
|
||||
|
||||
const LookbackNoLimit = abi.ChainEpoch(-1)
|
||||
@@ -77,6 +71,12 @@ type FullNode interface {
|
||||
// ChainHead returns the current head of the chain.
|
||||
ChainHead(context.Context) (*types.TipSet, error) //perm:read
|
||||
|
||||
// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
|
||||
ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
|
||||
|
||||
// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
|
||||
ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
|
||||
|
||||
// ChainGetBlock returns the block specified by the given CID.
|
||||
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) //perm:read
|
||||
// ChainGetTipSet returns the tipset specified by the given TipSetKey.
|
||||
@@ -113,11 +113,6 @@ type FullNode interface {
|
||||
// will be returned.
|
||||
ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
|
||||
|
||||
// ChainGetTipSetAfterHeight looks back for a tipset at the specified epoch.
|
||||
// If there are no blocks at the specified epoch, the first non-nil tipset at a later epoch
|
||||
// will be returned.
|
||||
ChainGetTipSetAfterHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
|
||||
|
||||
// ChainReadObj reads ipld nodes referenced by the specified CID from chain
|
||||
// blockstore and returns raw bytes.
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error) //perm:read
|
||||
@@ -128,9 +123,6 @@ type FullNode interface {
|
||||
// ChainHasObj checks if a given CID exists in the chain blockstore.
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
|
||||
|
||||
// ChainPutObj puts a given object into the block store
|
||||
ChainPutObj(context.Context, blocks.Block) error //perm:admin
|
||||
|
||||
// ChainStatObj returns statistics about the graph referenced by 'obj'.
|
||||
// If 'base' is also specified, then the returned stat will be a diff
|
||||
// between the two objects.
|
||||
@@ -152,7 +144,7 @@ type FullNode interface {
|
||||
|
||||
// ChainGetPath returns a set of revert/apply operations needed to get from
|
||||
// one tipset to another, for example:
|
||||
// ```
|
||||
//```
|
||||
// to
|
||||
// ^
|
||||
// from tAA
|
||||
@@ -161,7 +153,7 @@ type FullNode interface {
|
||||
// ^---*--^
|
||||
// ^
|
||||
// tRR
|
||||
// ```
|
||||
//```
|
||||
// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
|
||||
ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) //perm:read
|
||||
|
||||
@@ -172,24 +164,6 @@ type FullNode interface {
|
||||
// If oldmsgskip is set, messages from before the requested roots are also not included.
|
||||
ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
|
||||
|
||||
// ChainExportRangeInternal triggers the export of a chain
|
||||
// CAR-snapshot directly to disk. It is similar to ChainExport,
|
||||
// except, depending on options, the snapshot can include receipts,
|
||||
// messages and stateroots for the length between the specified head
|
||||
// and tail, thus producing "archival-grade" snapshots that include
|
||||
// all the on-chain data. The header chain is included back to
|
||||
// genesis and these snapshots can be used to initialize Filecoin
|
||||
// nodes.
|
||||
ChainExportRangeInternal(ctx context.Context, head, tail types.TipSetKey, cfg ChainExportConfig) error //perm:admin
|
||||
|
||||
// ChainPrune forces compaction on cold store and garbage collects; only supported if you
|
||||
// are using the splitstore
|
||||
ChainPrune(ctx context.Context, opts PruneOpts) error //perm:admin
|
||||
|
||||
// ChainHotGC does online (badger) GC on the hot store; only supported if you are using
|
||||
// the splitstore
|
||||
ChainHotGC(ctx context.Context, opts HotGCOpts) error //perm:admin
|
||||
|
||||
// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
|
||||
// if supported by the underlying implementation.
|
||||
ChainCheckBlockstore(context.Context) error //perm:admin
|
||||
@@ -197,8 +171,13 @@ type FullNode interface {
|
||||
// ChainBlockstoreInfo returns some basic information about the blockstore
|
||||
ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
|
||||
|
||||
// ChainGetEvents returns the events under an event AMT root CID.
|
||||
ChainGetEvents(context.Context, cid.Cid) ([]types.Event, error) //perm:read
|
||||
// MethodGroup: Beacon
|
||||
// The Beacon method group contains methods for interacting with the random beacon (DRAND)
|
||||
|
||||
// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
|
||||
// the entry has not yet been produced, the call will block until the entry
|
||||
// becomes available
|
||||
BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
|
||||
|
||||
// GasEstimateFeeCap estimates gas fee cap
|
||||
GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
|
||||
@@ -296,10 +275,8 @@ type FullNode interface {
|
||||
MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
|
||||
MpoolSub(context.Context) (<-chan MpoolUpdate, error) //perm:read
|
||||
|
||||
// MpoolClear clears pending messages from the mpool.
|
||||
// If clearLocal is true, ALL messages will be cleared.
|
||||
// If clearLocal is false, local messages will be protected, all others will be cleared.
|
||||
MpoolClear(ctx context.Context, clearLocal bool) error //perm:write
|
||||
// MpoolClear clears pending messages from the mpool
|
||||
MpoolClear(context.Context, bool) error //perm:write
|
||||
|
||||
// MpoolGetConfig returns (a copy of) the current mpool config
|
||||
MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read
|
||||
@@ -313,7 +290,7 @@ type FullNode interface {
|
||||
|
||||
// // UX ?
|
||||
|
||||
// MethodGroup: WalletF
|
||||
// MethodGroup: Wallet
|
||||
|
||||
// WalletNew creates a new address in the wallet with the given sigType.
|
||||
// Available key types: bls, secp256k1, secp256k1-ledger
|
||||
@@ -354,7 +331,7 @@ type FullNode interface {
|
||||
// ClientImport imports file under the specified path into filestore.
|
||||
ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin
|
||||
// ClientRemoveImport removes file import
|
||||
ClientRemoveImport(ctx context.Context, importID imports.ID) error //perm:admin
|
||||
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
|
||||
// ClientStartDeal proposes a deal with a miner.
|
||||
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin
|
||||
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
|
||||
@@ -374,17 +351,16 @@ type FullNode interface {
|
||||
// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
|
||||
ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read
|
||||
// ClientRetrieve initiates the retrieval of a file, as specified in the order.
|
||||
ClientRetrieve(ctx context.Context, params RetrievalOrder) (*RestrievalRes, error) //perm:admin
|
||||
// ClientRetrieveWait waits for retrieval to be complete
|
||||
ClientRetrieveWait(ctx context.Context, deal retrievalmarket.DealID) error //perm:admin
|
||||
// ClientExport exports a file stored in the local filestore to a system file
|
||||
ClientExport(ctx context.Context, exportRef ExportRef, fileRef FileRef) error //perm:admin
|
||||
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin
|
||||
// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
|
||||
// of status updates.
|
||||
ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
|
||||
// ClientListRetrievals returns information about retrievals made by the local client
|
||||
ClientListRetrievals(ctx context.Context) ([]RetrievalInfo, error) //perm:write
|
||||
// ClientGetRetrievalUpdates returns status of updated retrieval deals
|
||||
ClientGetRetrievalUpdates(ctx context.Context) (<-chan RetrievalInfo, error) //perm:write
|
||||
// ClientQueryAsk returns a signed StorageAsk from the specified miner.
|
||||
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*StorageAsk, error) //perm:read
|
||||
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
|
||||
// ClientCalcCommP calculates the CommP and data size of the specified CID
|
||||
ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error) //perm:read
|
||||
// ClientCalcCommP calculates the CommP for a specified file
|
||||
@@ -408,12 +384,12 @@ type FullNode interface {
|
||||
ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
|
||||
|
||||
// ClientUnimport removes references to the specified file from filestore
|
||||
// ClientUnimport(path string)
|
||||
//ClientUnimport(path string)
|
||||
|
||||
// ClientListImports lists imported files and their root CIDs
|
||||
ClientListImports(ctx context.Context) ([]Import, error) //perm:write
|
||||
|
||||
// ClientListAsks() []Ask
|
||||
//ClientListAsks() []Ask
|
||||
|
||||
// MethodGroup: State
|
||||
// The State methods are used to query, inspect, and interact with chain state.
|
||||
@@ -428,7 +404,7 @@ type FullNode interface {
|
||||
StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
|
||||
// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
|
||||
//
|
||||
// If a tipset key is provided, and a replacing message is not found on chain,
|
||||
// If a tipset key is provided, and a replacing message is found on chain,
|
||||
// the method will return an error saying that the message wasn't found
|
||||
//
|
||||
// If no tipset key is provided, the appropriate tipset is looked up, and if
|
||||
@@ -452,8 +428,6 @@ type FullNode interface {
|
||||
StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) //perm:read
|
||||
// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
|
||||
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) //perm:read
|
||||
// StateEncodeParams attempts to encode the provided json params to the binary from
|
||||
StateEncodeParams(ctx context.Context, toActCode cid.Cid, method abi.MethodNum, params json.RawMessage) ([]byte, error) //perm:read
|
||||
|
||||
// StateNetworkName returns the name of the network the node is synced to
|
||||
StateNetworkName(context.Context) (dtypes.NetworkName, error) //perm:read
|
||||
@@ -467,7 +441,7 @@ type FullNode interface {
|
||||
// StateMinerPower returns the power of the indicated miner
|
||||
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
|
||||
// StateMinerInfo returns info about the indicated miner
|
||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) //perm:read
|
||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
|
||||
// StateMinerDeadlines returns all the proving deadlines for the given miner
|
||||
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
|
||||
// StateMinerPartitions returns all partitions in the specified deadline
|
||||
@@ -484,15 +458,10 @@ type FullNode interface {
|
||||
StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
|
||||
// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
|
||||
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
|
||||
// StateMinerSectorAllocated checks if a sector number is marked as allocated.
|
||||
// StateMinerSectorAllocated checks if a sector is allocated
|
||||
StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
|
||||
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
|
||||
// Returns nil and no error if the sector isn't precommitted.
|
||||
//
|
||||
// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
|
||||
// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
|
||||
// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
|
||||
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
|
||||
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
|
||||
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
|
||||
// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
|
||||
// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
|
||||
// expiration epoch
|
||||
@@ -546,35 +515,18 @@ type FullNode interface {
|
||||
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
|
||||
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
|
||||
// StateMarketDeals returns information about every deal in the Storage Market
|
||||
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
|
||||
StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
|
||||
// StateMarketStorageDeal returns information about the indicated deal
|
||||
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
|
||||
// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal. Returns nil if
|
||||
// pending allocation is not found.
|
||||
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetAllocation returns the allocation for a given address and allocation ID.
|
||||
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetAllocations returns the all the allocations for a given client.
|
||||
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetClaim returns the claim for a given address and claim ID.
|
||||
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
|
||||
// StateGetClaims returns the all the claims for a given provider.
|
||||
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
|
||||
// StateComputeDataCID computes DataCID from a set of on-chain deals
|
||||
StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
|
||||
// StateLookupID retrieves the ID address of the given address
|
||||
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
|
||||
// StateAccountKey returns the public key address of the given ID address
|
||||
StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateLookupRobustAddress returns the public key address of the given ID address for non-account addresses (multisig, miners etc)
|
||||
StateLookupRobustAddress(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateChangedActors returns all the actors whose states change between the two given state CIDs
|
||||
// TODO: Should this take tipset keys instead?
|
||||
StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
|
||||
// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
|
||||
StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
|
||||
// StateMinerAllocated returns a bitfield containing all sector numbers marked as allocated in miner state
|
||||
StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) //perm:read
|
||||
// StateCompute is a flexible command that applies the given messages on the given tipset.
|
||||
// The messages are run as though the VM were at the provided height.
|
||||
//
|
||||
@@ -616,7 +568,7 @@ type FullNode interface {
|
||||
// Returns nil if there is no entry in the data cap table for the
|
||||
// address.
|
||||
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
|
||||
// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
|
||||
// StateVerifiedClientStatus returns the address of the Verified Registry's root key
|
||||
StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
|
||||
// can issue. It takes the deal size and verified status as parameters.
|
||||
@@ -630,28 +582,6 @@ type FullNode interface {
|
||||
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
|
||||
// StateNetworkVersion returns the network version at the given tipset
|
||||
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
|
||||
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
|
||||
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
|
||||
// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
|
||||
StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
|
||||
|
||||
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
|
||||
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
|
||||
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
|
||||
// StateGetRandomnessDigestFromTickets. is used to sample the chain for randomness.
|
||||
StateGetRandomnessDigestFromTickets(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
// StateGetRandomnessDigestFromBeacon is used to sample the beacon for randomness.
|
||||
StateGetRandomnessDigestFromBeacon(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
|
||||
// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
|
||||
// the entry has not yet been produced, the call will block until the entry
|
||||
// becomes available
|
||||
StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
|
||||
|
||||
// StateGetNetworkParams return current network params
|
||||
StateGetNetworkParams(ctx context.Context) (*NetworkParams, error) //perm:read
|
||||
|
||||
// MethodGroup: Msig
|
||||
// The Msig methods are used to interact with multisig wallets on the
|
||||
@@ -665,14 +595,14 @@ type FullNode interface {
|
||||
// It takes the following params: <multisig address>, <start epoch>, <end epoch>
|
||||
MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
|
||||
|
||||
// MsigGetPending returns pending transactions for the given multisig
|
||||
// wallet. Once pending transactions are fully approved, they will no longer
|
||||
// appear here.
|
||||
//MsigGetPending returns pending transactions for the given multisig
|
||||
//wallet. Once pending transactions are fully approved, they will no longer
|
||||
//appear here.
|
||||
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) //perm:read
|
||||
|
||||
// MsigCreate creates a multisig wallet
|
||||
// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
|
||||
// <initial balance>, <sender address of the create msg>, <gas price>
|
||||
//<initial balance>, <sender address of the create msg>, <gas price>
|
||||
MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (*MessagePrototype, error) //perm:sign
|
||||
|
||||
// MsigPropose proposes a multisig message
|
||||
@@ -692,14 +622,10 @@ type FullNode interface {
|
||||
// <sender address of the approve msg>, <method to call in the proposed message>, <params to include in the proposed message>
|
||||
MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
|
||||
|
||||
// MsigCancel cancels a previously-proposed multisig message
|
||||
// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
|
||||
MsigCancel(context.Context, address.Address, uint64, address.Address) (*MessagePrototype, error) //perm:sign
|
||||
|
||||
// MsigCancel cancels a previously-proposed multisig message
|
||||
// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
|
||||
// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
|
||||
MsigCancelTxnHash(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
|
||||
MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (*MessagePrototype, error) //perm:sign
|
||||
|
||||
// MsigAddPropose proposes adding a signer in the multisig
|
||||
// It takes the following params: <multisig address>, <sender address of the propose msg>,
|
||||
@@ -752,17 +678,7 @@ type FullNode interface {
|
||||
// MethodGroup: Paych
|
||||
// The Paych methods are for interacting with and managing payment channels
|
||||
|
||||
// PaychGet gets or creates a payment channel between address pair
|
||||
// The specified amount will be reserved for use. If there aren't enough non-reserved funds
|
||||
// available, funds will be added through an on-chain message.
|
||||
// - When opts.OffChain is true, this call will not cause any messages to be sent to the chain (no automatic
|
||||
// channel creation/funds adding). If the operation can't be performed without sending a message an error will be
|
||||
// returned. Note that even when this option is specified, this call can be blocked by previous operations on the
|
||||
// channel waiting for on-chain operations.
|
||||
PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt, opts PaychGetOpts) (*ChannelInfo, error) //perm:sign
|
||||
// PaychFund gets or creates a payment channel between address pair.
|
||||
// The specified amount will be added to the channel through on-chain send for future use
|
||||
PaychFund(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error) //perm:sign
|
||||
PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error) //perm:sign
|
||||
PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error) //perm:sign
|
||||
PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error) //perm:sign
|
||||
PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error) //perm:sign
|
||||
@@ -784,121 +700,11 @@ type FullNode interface {
|
||||
|
||||
NodeStatus(ctx context.Context, inclChainStatus bool) (NodeStatus, error) //perm:read
|
||||
|
||||
// MethodGroup: Eth
|
||||
// These methods are used for Ethereum-compatible JSON-RPC calls
|
||||
//
|
||||
// EthAccounts will always return [] since we don't expect Lotus to manage private keys
|
||||
EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) //perm:read
|
||||
// EthAddressToFilecoinAddress converts an EthAddress into an f410 Filecoin Address
|
||||
EthAddressToFilecoinAddress(ctx context.Context, ethAddress ethtypes.EthAddress) (address.Address, error) //perm:read
|
||||
// FilecoinAddressToEthAddress converts an f410 or f0 Filecoin Address to an EthAddress
|
||||
FilecoinAddressToEthAddress(ctx context.Context, filecoinAddress address.Address) (ethtypes.EthAddress, error) //perm:read
|
||||
// EthBlockNumber returns the height of the latest (heaviest) TipSet
|
||||
EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
|
||||
// EthGetBlockTransactionCountByNumber returns the number of messages in the TipSet
|
||||
EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error) //perm:read
|
||||
// EthGetBlockTransactionCountByHash returns the number of messages in the TipSet
|
||||
EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error) //perm:read
|
||||
|
||||
EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error) //perm:read
|
||||
EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error) //perm:read
|
||||
EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error) //perm:read
|
||||
EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error) //perm:read
|
||||
EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error) //perm:read
|
||||
EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error) //perm:read
|
||||
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthUint64, error) //perm:read
|
||||
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error) //perm:read
|
||||
EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error) //perm:read
|
||||
EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) //perm:read
|
||||
EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) //perm:read
|
||||
|
||||
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
|
||||
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
|
||||
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBigInt, error) //perm:read
|
||||
EthChainId(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
|
||||
EthSyncing(ctx context.Context) (ethtypes.EthSyncingResult, error) //perm:read
|
||||
NetVersion(ctx context.Context) (string, error) //perm:read
|
||||
NetListening(ctx context.Context) (bool, error) //perm:read
|
||||
EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
|
||||
EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error) //perm:read
|
||||
EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) //perm:read
|
||||
|
||||
EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error) //perm:read
|
||||
EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error) //perm:read
|
||||
EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
|
||||
|
||||
EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read
|
||||
|
||||
// Returns event logs matching given filter spec.
|
||||
EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error) //perm:read
|
||||
|
||||
// Polling method for a filter, returns event logs which occurred since last poll.
|
||||
// (requires write perm since timestamp of last filter execution will be written)
|
||||
EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:read
|
||||
|
||||
// Returns event logs matching filter with given id.
|
||||
// (requires write perm since timestamp of last filter execution will be written)
|
||||
EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:read
|
||||
|
||||
// Installs a persistent filter based on given filter spec.
|
||||
EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error) //perm:read
|
||||
|
||||
// Installs a persistent filter to notify when a new block arrives.
|
||||
EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:read
|
||||
|
||||
// Installs a persistent filter to notify when new messages arrive in the message pool.
|
||||
EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:read
|
||||
|
||||
// Uninstalls a filter with given id.
|
||||
EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error) //perm:read
|
||||
|
||||
// Subscribe to different event types using websockets
|
||||
// eventTypes is one or more of:
|
||||
// - newHeads: notify when new blocks arrive.
|
||||
// - pendingTransactions: notify when new messages arrive in the message pool.
|
||||
// - logs: notify new event logs that match a criteria
|
||||
// params contains additional parameters used with the log event type
|
||||
// The client will receive a stream of EthSubscriptionResponse values until EthUnsubscribe is called.
|
||||
EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error) //perm:read
|
||||
|
||||
// Unsubscribe from a websocket subscription
|
||||
EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error) //perm:read
|
||||
|
||||
// Returns the client version
|
||||
Web3ClientVersion(ctx context.Context) (string, error) //perm:read
|
||||
|
||||
// TraceAPI related methods
|
||||
//
|
||||
// Returns traces created at given block
|
||||
EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error) //perm:read
|
||||
// Replays all transactions in a block returning the requested traces for each transaction
|
||||
EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) //perm:read
|
||||
|
||||
// go-ethereum "debug" trace endpoints
|
||||
EthDebugTraceCall(ctx context.Context, args ethtypes.EthTxArgs, blockNrOrHash ethtypes.EthBlockNumberOrHash, config *ethtypes.TraceCallConfig) (interface{}, error)
|
||||
EthDebugTraceBlockByNumber(ctx context.Context, number string, config *ethtypes.TraceConfig) ([]*ethtypes.TxTraceResult, error)
|
||||
EthDebugTraceBlockByHash(ctx context.Context, hash ethtypes.EthHash, config *ethtypes.TraceConfig) ([]*ethtypes.TxTraceResult, error)
|
||||
|
||||
// CreateBackup creates node backup onder the specified file name. The
|
||||
// method requires that the lotus daemon is running with the
|
||||
// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
|
||||
// the path specified when calling CreateBackup is within the base path
|
||||
CreateBackup(ctx context.Context, fpath string) error //perm:admin
|
||||
|
||||
RaftState(ctx context.Context) (*RaftStateData, error) //perm:read
|
||||
RaftLeader(ctx context.Context) (peer.ID, error) //perm:read
|
||||
}
|
||||
|
||||
// reverse interface to the client, called after EthSubscribe
|
||||
type EthSubscriber interface {
|
||||
// note: the parameter is ethtypes.EthSubscriptionResponse serialized as json object
|
||||
EthSubscription(ctx context.Context, r jsonrpc.RawParams) error // rpc_method:eth_subscription notify:true
|
||||
}
|
||||
|
||||
type StorageAsk struct {
|
||||
Response *storagemarket.StorageAsk
|
||||
|
||||
DealProtocols []string
|
||||
}
|
||||
|
||||
type FileRef struct {
|
||||
@@ -917,28 +723,17 @@ type MinerSectors struct {
|
||||
|
||||
type ImportRes struct {
|
||||
Root cid.Cid
|
||||
ImportID imports.ID
|
||||
ImportID importmgr.ImportID
|
||||
}
|
||||
|
||||
type Import struct {
|
||||
Key imports.ID
|
||||
Key importmgr.ImportID
|
||||
Err string
|
||||
|
||||
Root *cid.Cid
|
||||
|
||||
// Source is the provenance of the import, e.g. "import", "unknown", else.
|
||||
// Currently useless but may be used in the future.
|
||||
Source string
|
||||
|
||||
// FilePath is the path of the original file. It is important that the file
|
||||
// is retained at this path, because it will be referenced during
|
||||
// the transfer (when we do the UnixFS chunking, we don't duplicate the
|
||||
// leaves, but rather point to chunks of the original data through
|
||||
// positional references).
|
||||
FilePath string
|
||||
|
||||
// CARPath is the path of the CAR file containing the DAG for this import.
|
||||
CARPath string
|
||||
Root *cid.Cid
|
||||
Source string
|
||||
FilePath string
|
||||
CARv2FilePath string
|
||||
}
|
||||
|
||||
type DealInfo struct {
|
||||
@@ -1011,10 +806,6 @@ const (
|
||||
PCHOutbound
|
||||
)
|
||||
|
||||
type PaychGetOpts struct {
|
||||
OffChain bool
|
||||
}
|
||||
|
||||
type PaychStatus struct {
|
||||
ControlAddr address.Address
|
||||
Direction PCHDir
|
||||
@@ -1032,23 +823,16 @@ type ChannelAvailableFunds struct {
|
||||
From address.Address
|
||||
// To is the to address of the channel
|
||||
To address.Address
|
||||
|
||||
// ConfirmedAmt is the total amount of funds that have been confirmed on-chain for the channel
|
||||
// ConfirmedAmt is the amount of funds that have been confirmed on-chain
|
||||
// for the channel
|
||||
ConfirmedAmt types.BigInt
|
||||
// PendingAmt is the amount of funds that are pending confirmation on-chain
|
||||
PendingAmt types.BigInt
|
||||
|
||||
// NonReservedAmt is part of ConfirmedAmt that is available for use (e.g. when the payment channel was pre-funded)
|
||||
NonReservedAmt types.BigInt
|
||||
// PendingAvailableAmt is the amount of funds that are pending confirmation on-chain that will become available once confirmed
|
||||
PendingAvailableAmt types.BigInt
|
||||
|
||||
// PendingWaitSentinel can be used with PaychGetWaitReady to wait for
|
||||
// confirmation of pending funds
|
||||
PendingWaitSentinel *cid.Cid
|
||||
// QueuedAmt is the amount that is queued up behind a pending request
|
||||
QueuedAmt types.BigInt
|
||||
|
||||
// VoucherRedeemedAmt is the amount that is redeemed by vouchers on-chain
|
||||
// and in the local datastore
|
||||
VoucherReedeemedAmt types.BigInt
|
||||
@@ -1094,7 +878,6 @@ type QueryOffer struct {
|
||||
Size uint64
|
||||
MinPrice types.BigInt
|
||||
UnsealPrice types.BigInt
|
||||
PricePerByte abi.TokenAmount
|
||||
PaymentInterval uint64
|
||||
PaymentIntervalIncrease uint64
|
||||
Miner address.Address
|
||||
@@ -1128,26 +911,22 @@ type MarketDeal struct {
|
||||
}
|
||||
|
||||
type RetrievalOrder struct {
|
||||
Root cid.Cid
|
||||
Piece *cid.Cid
|
||||
DataSelector *Selector
|
||||
|
||||
// todo: Size/Total are only used for calculating price per byte; we should let users just pass that
|
||||
// TODO: make this less unixfs specific
|
||||
Root cid.Cid
|
||||
Piece *cid.Cid
|
||||
Size uint64
|
||||
Total types.BigInt
|
||||
|
||||
LocalCARV2FilePath string // if specified, get data from a local CARv2 file.
|
||||
// TODO: support offset
|
||||
Total types.BigInt
|
||||
UnsealPrice types.BigInt
|
||||
PaymentInterval uint64
|
||||
PaymentIntervalIncrease uint64
|
||||
Client address.Address
|
||||
Miner address.Address
|
||||
MinerPeer *retrievalmarket.RetrievalPeer
|
||||
|
||||
RemoteStore *RemoteStoreID `json:"RemoteStore,omitempty"`
|
||||
}
|
||||
|
||||
type RemoteStoreID = uuid.UUID
|
||||
|
||||
type InvocResult struct {
|
||||
MsgCid cid.Cid
|
||||
Msg *types.Message
|
||||
@@ -1282,7 +1061,7 @@ type CirculatingSupply struct {
|
||||
type MiningBaseInfo struct {
|
||||
MinerPower types.BigInt
|
||||
NetworkPower types.BigInt
|
||||
Sectors []builtin.ExtendedSectorInfo
|
||||
Sectors []builtin.SectorInfo
|
||||
WorkerKey address.Address
|
||||
SectorSize abi.SectorSize
|
||||
PrevBeaconEntry types.BeaconEntry
|
||||
@@ -1373,32 +1152,3 @@ type MsigTransaction struct {
|
||||
|
||||
Approved []address.Address
|
||||
}
|
||||
|
||||
type PruneOpts struct {
|
||||
MovingGC bool
|
||||
RetainState int64
|
||||
}
|
||||
|
||||
type HotGCOpts struct {
|
||||
Threshold float64
|
||||
Periodic bool
|
||||
Moving bool
|
||||
}
|
||||
|
||||
type EthTxReceipt struct {
|
||||
TransactionHash ethtypes.EthHash `json:"transactionHash"`
|
||||
TransactionIndex ethtypes.EthUint64 `json:"transactionIndex"`
|
||||
BlockHash ethtypes.EthHash `json:"blockHash"`
|
||||
BlockNumber ethtypes.EthUint64 `json:"blockNumber"`
|
||||
From ethtypes.EthAddress `json:"from"`
|
||||
To *ethtypes.EthAddress `json:"to"`
|
||||
StateRoot ethtypes.EthHash `json:"root"`
|
||||
Status ethtypes.EthUint64 `json:"status"`
|
||||
ContractAddress *ethtypes.EthAddress `json:"contractAddress"`
|
||||
CumulativeGasUsed ethtypes.EthUint64 `json:"cumulativeGasUsed"`
|
||||
GasUsed ethtypes.EthUint64 `json:"gasUsed"`
|
||||
EffectiveGasPrice ethtypes.EthBigInt `json:"effectiveGasPrice"`
|
||||
LogsBloom ethtypes.EthBytes `json:"logsBloom"`
|
||||
Logs []ethtypes.EthLog `json:"logs"`
|
||||
Type ethtypes.EthUint64 `json:"type"`
|
||||
}
|
||||
|
||||
+4
-76
@@ -3,20 +3,15 @@ package api
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/types/ethtypes"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -27,109 +22,42 @@ import (
|
||||
// When adding / changing methods in this file:
|
||||
// * Do the change here
|
||||
// * Adjust implementation in `node/impl/`
|
||||
// * Run `make clean && make deps && make gen` - this will:
|
||||
// * Run `make gen` - this will:
|
||||
// * Generate proxy structs
|
||||
// * Generate mocks
|
||||
// * Generate markdown docs
|
||||
// * Generate openrpc blobs
|
||||
|
||||
type Gateway interface {
|
||||
MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
|
||||
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
|
||||
MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*MiningBaseInfo, error)
|
||||
StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error)
|
||||
GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
|
||||
StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
ChainPutObj(context.Context, blocks.Block) error
|
||||
ChainHead(ctx context.Context) (*types.TipSet, error)
|
||||
ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
|
||||
ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
|
||||
ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
|
||||
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
|
||||
ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*HeadChange, error)
|
||||
ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
|
||||
ChainNotify(context.Context) (<-chan []*HeadChange, error)
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainGetGenesis(context.Context) (*types.TipSet, error)
|
||||
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
|
||||
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
|
||||
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
|
||||
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
|
||||
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
|
||||
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
|
||||
MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
|
||||
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
|
||||
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error)
|
||||
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
|
||||
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
|
||||
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
|
||||
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
|
||||
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
|
||||
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
|
||||
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
|
||||
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
|
||||
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
|
||||
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
|
||||
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
|
||||
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
|
||||
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
|
||||
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
|
||||
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
|
||||
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
|
||||
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
|
||||
StateNetworkName(context.Context) (dtypes.NetworkName, error)
|
||||
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
|
||||
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
|
||||
StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
|
||||
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
|
||||
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
|
||||
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
|
||||
WalletBalance(context.Context, address.Address) (types.BigInt, error)
|
||||
Version(context.Context) (APIVersion, error)
|
||||
Discover(context.Context) (apitypes.OpenRPCDocument, error)
|
||||
|
||||
EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error)
|
||||
EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error)
|
||||
EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error)
|
||||
EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error)
|
||||
EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error)
|
||||
EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error)
|
||||
EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error)
|
||||
EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error)
|
||||
EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error)
|
||||
EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error)
|
||||
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthUint64, error)
|
||||
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error)
|
||||
EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error)
|
||||
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
|
||||
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
|
||||
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBigInt, error)
|
||||
EthChainId(ctx context.Context) (ethtypes.EthUint64, error)
|
||||
EthSyncing(ctx context.Context) (ethtypes.EthSyncingResult, error)
|
||||
NetVersion(ctx context.Context) (string, error)
|
||||
NetListening(ctx context.Context) (bool, error)
|
||||
EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error)
|
||||
EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
|
||||
EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
|
||||
EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
|
||||
EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
|
||||
EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
|
||||
EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
|
||||
EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)
|
||||
EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
|
||||
EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
|
||||
EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error)
|
||||
EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error)
|
||||
EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error)
|
||||
EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error)
|
||||
EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error)
|
||||
EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error)
|
||||
Web3ClientVersion(ctx context.Context) (string, error)
|
||||
EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error)
|
||||
EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error)
|
||||
EthDebugTraceCall(ctx context.Context, args ethtypes.EthTxArgs, blockNrOrHash ethtypes.EthBlockNumberOrHash, config *ethtypes.TraceCallConfig) (interface{}, error)
|
||||
EthDebugTraceBlockByNumber(ctx context.Context, number string, config *ethtypes.TraceConfig) ([]*ethtypes.TxTraceResult, error)
|
||||
EthDebugTraceBlockByHash(ctx context.Context, hash ethtypes.EthHash, config *ethtypes.TraceConfig) ([]*ethtypes.TxTraceResult, error)
|
||||
}
|
||||
|
||||
+5
-16
@@ -2,12 +2,11 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/metrics"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/protocol"
|
||||
metrics "github.com/libp2p/go-libp2p-core/metrics"
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/protocol"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -26,7 +25,6 @@ type Net interface {
|
||||
|
||||
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
|
||||
NetPeers(context.Context) ([]peer.AddrInfo, error) //perm:read
|
||||
NetPing(context.Context, peer.ID) (time.Duration, error) //perm:read
|
||||
NetConnect(context.Context, peer.AddrInfo) error //perm:write
|
||||
NetAddrsListen(context.Context) (peer.AddrInfo, error) //perm:read
|
||||
NetDisconnect(context.Context, peer.ID) error //perm:write
|
||||
@@ -53,15 +51,6 @@ type Net interface {
|
||||
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
|
||||
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
|
||||
|
||||
NetProtectAdd(ctx context.Context, acl []peer.ID) error //perm:admin
|
||||
NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
|
||||
NetProtectList(ctx context.Context) ([]peer.ID, error) //perm:read
|
||||
|
||||
// ResourceManager API
|
||||
NetStat(ctx context.Context, scope string) (NetStat, error) //perm:read
|
||||
NetLimit(ctx context.Context, scope string) (NetLimit, error) //perm:read
|
||||
NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
|
||||
|
||||
// ID returns peerID of libp2p node backing this API
|
||||
ID(context.Context) (peer.ID, error) //perm:read
|
||||
}
|
||||
@@ -73,5 +62,5 @@ type CommonNet interface {
|
||||
|
||||
type NatInfo struct {
|
||||
Reachability network.Reachability
|
||||
PublicAddrs []string
|
||||
PublicAddr string
|
||||
}
|
||||
|
||||
+63
-281
@@ -5,28 +5,27 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
|
||||
"github.com/filecoin-project/go-fil-markets/piecestore"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v9/market"
|
||||
abinetwork "github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
builtinactors "github.com/filecoin-project/lotus/chain/actors/builtin"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -50,22 +49,8 @@ type StorageMiner interface {
|
||||
ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
|
||||
ActorAddressConfig(ctx context.Context) (AddressConfig, error) //perm:read
|
||||
|
||||
// WithdrawBalance allows to withdraw balance from miner actor to owner address
|
||||
// Specify amount as "0" to withdraw full balance. This method returns a message CID
|
||||
// and does not wait for message execution
|
||||
ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
|
||||
|
||||
// BeneficiaryWithdrawBalance allows the beneficiary of a miner to withdraw balance from miner actor
|
||||
// Specify amount as "0" to withdraw full balance. This method returns a message CID
|
||||
// and does not wait for message execution
|
||||
BeneficiaryWithdrawBalance(context.Context, abi.TokenAmount) (cid.Cid, error) //perm:admin
|
||||
|
||||
MiningBase(context.Context) (*types.TipSet, error) //perm:read
|
||||
|
||||
ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
|
||||
|
||||
ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
|
||||
|
||||
// Temp api for testing
|
||||
PledgeSector(context.Context) (abi.SectorID, error) //perm:write
|
||||
|
||||
@@ -75,9 +60,9 @@ type StorageMiner interface {
|
||||
// Add piece to an open sector. If no sectors with enough space are open,
|
||||
// either a new sector will be created, or this call will block until more
|
||||
// sectors can be created.
|
||||
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
|
||||
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
|
||||
|
||||
SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
|
||||
SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
|
||||
|
||||
// List all staged sectors
|
||||
SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
|
||||
@@ -106,8 +91,7 @@ type StorageMiner interface {
|
||||
SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error //perm:admin
|
||||
// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
|
||||
// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
|
||||
SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
|
||||
SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber, snap bool) error //perm:admin
|
||||
SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
|
||||
// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
|
||||
// automatically removes it from storage
|
||||
SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
|
||||
@@ -115,7 +99,8 @@ type StorageMiner interface {
|
||||
// Returns null if message wasn't sent
|
||||
SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
|
||||
// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
|
||||
SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
|
||||
SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
|
||||
SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
|
||||
// SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
|
||||
// Returns null if message wasn't sent
|
||||
SectorPreCommitFlush(ctx context.Context) ([]sealiface.PreCommitBatchRes, error) //perm:admin
|
||||
@@ -126,99 +111,47 @@ type StorageMiner interface {
|
||||
SectorCommitFlush(ctx context.Context) ([]sealiface.CommitBatchRes, error) //perm:admin
|
||||
// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
|
||||
SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
|
||||
SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
|
||||
// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
|
||||
SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
|
||||
// SectorUnseal unseals the provided sector
|
||||
SectorUnseal(ctx context.Context, number abi.SectorNumber) error //perm:admin
|
||||
|
||||
// SectorNumAssignerMeta returns sector number assigner metadata - reserved/allocated
|
||||
SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
|
||||
// SectorNumReservations returns a list of sector number reservations
|
||||
SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
|
||||
// SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
|
||||
// numbers or name. Set force to true to override safety checks.
|
||||
// Valid characters for name: a-z, A-Z, 0-9, _, -
|
||||
SectorNumReserve(ctx context.Context, name string, sectors bitfield.BitField, force bool) error //perm:admin
|
||||
// SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
|
||||
// by default lotus will allocate lowest-available sector numbers to the reservation.
|
||||
// For restrictions on `name` see SectorNumReserve
|
||||
SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) //perm:admin
|
||||
// SectorNumFree drops a sector reservation
|
||||
SectorNumFree(ctx context.Context, name string) error //perm:admin
|
||||
|
||||
SectorReceive(ctx context.Context, meta RemoteSectorMeta) error //perm:admin
|
||||
|
||||
// WorkerConnect tells the node to connect to workers RPC
|
||||
WorkerConnect(context.Context, string) error //perm:admin retry:true
|
||||
WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
|
||||
WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) //perm:admin
|
||||
|
||||
// storiface.WorkerReturn
|
||||
ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnDownloadSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
//storiface.WorkerReturn
|
||||
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
|
||||
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
|
||||
|
||||
// SealingSchedDiag dumps internal sealing scheduler state
|
||||
SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
|
||||
SealingAbort(ctx context.Context, call storiface.CallID) error //perm:admin
|
||||
// SealingSchedRemove removes a request from sealing pipeline
|
||||
SealingRemoveRequest(ctx context.Context, schedId uuid.UUID) error //perm:admin
|
||||
|
||||
// paths.SectorIndex
|
||||
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
|
||||
StorageDetach(ctx context.Context, id storiface.ID, url string) error //perm:admin
|
||||
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
|
||||
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
|
||||
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
|
||||
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
|
||||
// StorageFindSector returns list of paths where the specified sector files exist.
|
||||
//
|
||||
// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
|
||||
// - Paths which have sector files locally (don't require fetching) will be listed first.
|
||||
// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
|
||||
// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
|
||||
// file types are specified, each type will be considered individually, and a union of all paths
|
||||
// which can accommodate each file type will be returned.
|
||||
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
|
||||
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
|
||||
// Paths with more weight and more % of free space are preferred.
|
||||
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
|
||||
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
|
||||
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
|
||||
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
|
||||
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
|
||||
//stores.SectorIndex
|
||||
StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error //perm:admin
|
||||
StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error) //perm:admin
|
||||
StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error //perm:admin
|
||||
StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
|
||||
StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
|
||||
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error) //perm:admin
|
||||
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
|
||||
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
|
||||
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
|
||||
|
||||
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
|
||||
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
|
||||
StorageLocal(ctx context.Context) (map[stores.ID]string, error) //perm:admin
|
||||
StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
|
||||
|
||||
StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
|
||||
|
||||
StorageAddLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
|
||||
|
||||
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
|
||||
MarketListDeals(ctx context.Context) ([]*MarketDeal, error) //perm:read
|
||||
|
||||
// MarketListRetrievalDeals is deprecated, returns empty list
|
||||
MarketListRetrievalDeals(ctx context.Context) ([]struct{}, error) //perm:read
|
||||
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
|
||||
MarketListDeals(ctx context.Context) ([]MarketDeal, error) //perm:read
|
||||
MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) //perm:read
|
||||
MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) //perm:read
|
||||
MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) //perm:read
|
||||
MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
|
||||
@@ -227,15 +160,12 @@ type StorageMiner interface {
|
||||
MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) //perm:read
|
||||
MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) //perm:write
|
||||
MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) //perm:write
|
||||
// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
|
||||
MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
|
||||
// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
|
||||
MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
|
||||
// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
|
||||
MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
|
||||
MarketPendingDeals(ctx context.Context) (PendingDealInfo, error) //perm:write
|
||||
MarketPublishPendingDeals(ctx context.Context) error //perm:admin
|
||||
MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error //perm:admin
|
||||
|
||||
// DagstoreListShards returns information about all shards known to the
|
||||
// DAG store. Only available on nodes running the markets subsystem.
|
||||
@@ -279,25 +209,12 @@ type StorageMiner interface {
|
||||
// DagstoreGC runs garbage collection on the DAG store.
|
||||
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
|
||||
|
||||
// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
|
||||
DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
|
||||
|
||||
// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
|
||||
// so they can download its index
|
||||
IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
|
||||
|
||||
// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
|
||||
IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
|
||||
|
||||
// DagstoreLookupPieces returns information about shards that contain the given CID.
|
||||
DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
|
||||
|
||||
// RuntimeSubsystems returns the subsystems that are enabled
|
||||
// in this instance.
|
||||
RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
|
||||
|
||||
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
|
||||
DealsList(ctx context.Context) ([]*MarketDeal, error) //perm:admin
|
||||
DealsList(ctx context.Context) ([]MarketDeal, error) //perm:admin
|
||||
DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin
|
||||
DealsSetConsiderOnlineStorageDeals(context.Context, bool) error //perm:admin
|
||||
DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error) //perm:admin
|
||||
@@ -313,6 +230,8 @@ type StorageMiner interface {
|
||||
DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error) //perm:admin
|
||||
DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin
|
||||
|
||||
StorageAddLocal(ctx context.Context, path string) error //perm:admin
|
||||
|
||||
PiecesListPieces(ctx context.Context) ([]cid.Cid, error) //perm:read
|
||||
PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) //perm:read
|
||||
PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
|
||||
@@ -324,17 +243,13 @@ type StorageMiner interface {
|
||||
// the path specified when calling CreateBackup is within the base path
|
||||
CreateBackup(ctx context.Context, fpath string) error //perm:admin
|
||||
|
||||
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef) (map[abi.SectorNumber]string, error) //perm:admin
|
||||
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
|
||||
|
||||
ComputeProof(ctx context.Context, ssi []builtinactors.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtinactors.PoStProof, error) //perm:read
|
||||
|
||||
// RecoverFault can be used to declare recoveries manually. It sends messages
|
||||
// to the miner actor with details of recovered sectors and returns the CID of messages. It honors the
|
||||
// maxPartitionsPerRecoveryMessage from the config
|
||||
RecoverFault(ctx context.Context, sectors []abi.SectorNumber) ([]cid.Cid, error) //perm:admin
|
||||
ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
|
||||
}
|
||||
|
||||
var _ storiface.WorkerReturn = *new(StorageMiner)
|
||||
var _ stores.SectorIndex = *new(StorageMiner)
|
||||
|
||||
type SealRes struct {
|
||||
Err string
|
||||
@@ -352,26 +267,19 @@ type SectorLog struct {
|
||||
Message string
|
||||
}
|
||||
|
||||
type SectorPiece struct {
|
||||
Piece abi.PieceInfo
|
||||
DealInfo *PieceDealInfo // nil for pieces which do not appear in deals (e.g. filler pieces)
|
||||
}
|
||||
|
||||
type SectorInfo struct {
|
||||
SectorID abi.SectorNumber
|
||||
State SectorState
|
||||
CommD *cid.Cid
|
||||
CommR *cid.Cid
|
||||
Proof []byte
|
||||
Deals []abi.DealID
|
||||
Pieces []SectorPiece
|
||||
Ticket SealTicket
|
||||
Seed SealSeed
|
||||
PreCommitMsg *cid.Cid
|
||||
CommitMsg *cid.Cid
|
||||
Retries uint64
|
||||
ToUpgrade bool
|
||||
ReplicaUpdateMessage *cid.Cid
|
||||
SectorID abi.SectorNumber
|
||||
State SectorState
|
||||
CommD *cid.Cid
|
||||
CommR *cid.Cid
|
||||
Proof []byte
|
||||
Deals []abi.DealID
|
||||
Ticket SealTicket
|
||||
Seed SealSeed
|
||||
PreCommitMsg *cid.Cid
|
||||
CommitMsg *cid.Cid
|
||||
Retries uint64
|
||||
ToUpgrade bool
|
||||
|
||||
LastErr string
|
||||
|
||||
@@ -421,10 +329,6 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
|
||||
|
||||
type SectorState string
|
||||
|
||||
func (s *SectorState) String() string {
|
||||
return string(*s)
|
||||
}
|
||||
|
||||
type AddrUse int
|
||||
|
||||
const (
|
||||
@@ -461,15 +365,10 @@ type SectorOffset struct {
|
||||
|
||||
// DealInfo is a tuple of deal identity and its schedule
|
||||
type PieceDealInfo struct {
|
||||
// "Old" builtin-market deal info
|
||||
PublishCid *cid.Cid
|
||||
DealID abi.DealID
|
||||
DealProposal *market.DealProposal
|
||||
|
||||
// Common deal info
|
||||
DealSchedule DealSchedule
|
||||
|
||||
// Best-effort deal asks
|
||||
KeepUnsealed bool
|
||||
}
|
||||
|
||||
@@ -499,7 +398,6 @@ type DagstoreShardResult struct {
|
||||
|
||||
type DagstoreInitializeAllParams struct {
|
||||
MaxConcurrency int
|
||||
IncludeSealed bool
|
||||
}
|
||||
|
||||
// DagstoreInitializeAllEvent represents an initialization event.
|
||||
@@ -511,119 +409,3 @@ type DagstoreInitializeAllEvent struct {
|
||||
Total int
|
||||
Current int
|
||||
}
|
||||
|
||||
type NumAssignerMeta struct {
|
||||
Reserved bitfield.BitField
|
||||
Allocated bitfield.BitField
|
||||
|
||||
// ChainAllocated+Reserved+Allocated
|
||||
InUse bitfield.BitField
|
||||
|
||||
Next abi.SectorNumber
|
||||
}
|
||||
|
||||
type RemoteSectorMeta struct {
|
||||
////////
|
||||
// BASIC SECTOR INFORMATION
|
||||
|
||||
// State specifies the first state the sector will enter after being imported
|
||||
// Must be one of the following states:
|
||||
// * Packing
|
||||
// * GetTicket
|
||||
// * PreCommitting
|
||||
// * SubmitCommit
|
||||
// * Proving/Available
|
||||
State SectorState
|
||||
|
||||
Sector abi.SectorID
|
||||
Type abi.RegisteredSealProof
|
||||
|
||||
////////
|
||||
// SEALING METADATA
|
||||
// (allows lotus to continue the sealing process)
|
||||
|
||||
// Required in Packing and later
|
||||
Pieces []SectorPiece // todo better type?
|
||||
|
||||
// Required in PreCommitting and later
|
||||
TicketValue abi.SealRandomness
|
||||
TicketEpoch abi.ChainEpoch
|
||||
PreCommit1Out storiface.PreCommit1Out // todo specify better
|
||||
|
||||
CommD *cid.Cid
|
||||
CommR *cid.Cid // SectorKey
|
||||
|
||||
// Required in SubmitCommit and later
|
||||
PreCommitInfo *miner.SectorPreCommitInfo
|
||||
PreCommitDeposit *big.Int
|
||||
PreCommitMessage *cid.Cid
|
||||
PreCommitTipSet types.TipSetKey
|
||||
|
||||
SeedValue abi.InteractiveSealRandomness
|
||||
SeedEpoch abi.ChainEpoch
|
||||
|
||||
CommitProof []byte
|
||||
|
||||
// Required in Proving/Available
|
||||
CommitMessage *cid.Cid
|
||||
|
||||
// Optional sector metadata to import
|
||||
Log []SectorLog
|
||||
|
||||
////////
|
||||
// SECTOR DATA SOURCE
|
||||
|
||||
// Sector urls - lotus will use those for fetching files into local storage
|
||||
|
||||
// Required in all states
|
||||
DataUnsealed *storiface.SectorLocation
|
||||
|
||||
// Required in PreCommitting and later
|
||||
DataSealed *storiface.SectorLocation
|
||||
DataCache *storiface.SectorLocation
|
||||
|
||||
////////
|
||||
// SEALING SERVICE HOOKS
|
||||
|
||||
// URL
|
||||
// RemoteCommit1Endpoint is an URL of POST endpoint which lotus will call requesting Commit1 (seal_commit_phase1)
|
||||
// request body will be json-serialized RemoteCommit1Params struct
|
||||
RemoteCommit1Endpoint string
|
||||
|
||||
// RemoteCommit2Endpoint is an URL of POST endpoint which lotus will call requesting Commit2 (seal_commit_phase2)
|
||||
// request body will be json-serialized RemoteCommit2Params struct
|
||||
RemoteCommit2Endpoint string
|
||||
|
||||
// RemoteSealingDoneEndpoint is called after the sector exists the sealing pipeline
|
||||
// request body will be json-serialized RemoteSealingDoneParams struct
|
||||
RemoteSealingDoneEndpoint string
|
||||
}
|
||||
|
||||
type RemoteCommit1Params struct {
|
||||
Ticket, Seed []byte
|
||||
|
||||
Unsealed cid.Cid
|
||||
Sealed cid.Cid
|
||||
|
||||
ProofType abi.RegisteredSealProof
|
||||
}
|
||||
|
||||
type RemoteCommit2Params struct {
|
||||
Sector abi.SectorID
|
||||
ProofType abi.RegisteredSealProof
|
||||
|
||||
// todo spec better
|
||||
Commit1Out storiface.Commit1Out
|
||||
}
|
||||
|
||||
type RemoteSealingDoneParams struct {
|
||||
// Successful is true if the sector has entered state considered as "successfully sealed"
|
||||
Successful bool
|
||||
|
||||
// State is the state the sector has entered
|
||||
// For example "Proving" / "Removing"
|
||||
State string
|
||||
|
||||
// Optional commit message CID
|
||||
CommitMessage *cid.Cid
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// stm: #unit
|
||||
package api
|
||||
|
||||
import (
|
||||
@@ -12,9 +11,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
)
|
||||
|
||||
func goCmd() string {
|
||||
@@ -30,7 +26,6 @@ func goCmd() string {
|
||||
}
|
||||
|
||||
func TestDoesntDependOnFFI(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_001
|
||||
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -43,7 +38,6 @@ func TestDoesntDependOnFFI(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDoesntDependOnBuild(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_FFI_DEPENDENCE_002
|
||||
deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -56,7 +50,6 @@ func TestDoesntDependOnBuild(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReturnTypes(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_001
|
||||
errType := reflect.TypeOf(new(error)).Elem()
|
||||
bareIface := reflect.TypeOf(new(interface{})).Elem()
|
||||
jmarsh := reflect.TypeOf(new(json.Marshaler)).Elem()
|
||||
@@ -122,23 +115,7 @@ func TestReturnTypes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPermTags(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_PERM_TAGS_001
|
||||
_ = PermissionedFullAPI(&FullNodeStruct{})
|
||||
_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
|
||||
_ = PermissionedWorkerAPI(&WorkerStruct{})
|
||||
}
|
||||
|
||||
func TestRetryErrorIsInTrue(t *testing.T) {
|
||||
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
|
||||
require.True(t, ErrorIsIn(&jsonrpc.RPCConnectionError{}, errorsToRetry))
|
||||
}
|
||||
|
||||
func TestRetryErrorIsInFalse(t *testing.T) {
|
||||
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
|
||||
require.False(t, ErrorIsIn(xerrors.Errorf("random error"), errorsToRetry))
|
||||
}
|
||||
|
||||
func TestRetryWrappedErrorIsInTrue(t *testing.T) {
|
||||
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
|
||||
require.True(t, ErrorIsIn(xerrors.Errorf("wrapped: %w", &jsonrpc.RPCConnectionError{}), errorsToRetry))
|
||||
}
|
||||
|
||||
+16
-34
@@ -7,10 +7,10 @@ import (
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/proof"
|
||||
|
||||
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
"github.com/filecoin-project/specs-storage/storage"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -29,30 +29,20 @@ type Worker interface {
|
||||
|
||||
// TaskType -> Weight
|
||||
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
|
||||
Paths(context.Context) ([]storiface.StoragePath, error) //perm:admin
|
||||
Paths(context.Context) ([]stores.StoragePath, error) //perm:admin
|
||||
Info(context.Context) (storiface.WorkerInfo, error) //perm:admin
|
||||
|
||||
// storiface.WorkerCalls
|
||||
DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
|
||||
AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
|
||||
SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
|
||||
SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error) //perm:admin
|
||||
SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
|
||||
SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error) //perm:admin
|
||||
FinalizeSector(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) //perm:admin
|
||||
FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) //perm:admin
|
||||
ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
|
||||
ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin
|
||||
ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin
|
||||
GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error) //perm:admin
|
||||
ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) //perm:admin
|
||||
MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
|
||||
UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
|
||||
Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
|
||||
DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error) //perm:admin
|
||||
|
||||
GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error) //perm:admin
|
||||
GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
|
||||
AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error) //perm:admin
|
||||
SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
|
||||
SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error) //perm:admin
|
||||
SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
|
||||
SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error) //perm:admin
|
||||
FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
|
||||
ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error) //perm:admin
|
||||
MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
|
||||
UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
|
||||
Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
|
||||
|
||||
TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
|
||||
TaskEnable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
|
||||
@@ -60,11 +50,7 @@ type Worker interface {
|
||||
// Storage / Other
|
||||
Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
|
||||
|
||||
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
|
||||
StorageAddLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
|
||||
StorageDetachAll(ctx context.Context) error //perm:admin
|
||||
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
|
||||
StorageAddLocal(ctx context.Context, path string) error //perm:admin
|
||||
|
||||
// SetEnabled marks the worker as enabled/disabled. Not that this setting
|
||||
// may take a few seconds to propagate to task scheduler
|
||||
@@ -81,10 +67,6 @@ type Worker interface {
|
||||
|
||||
// Like ProcessSession, but returns an error when worker is disabled
|
||||
Session(context.Context) (uuid.UUID, error) //perm:admin
|
||||
|
||||
// Trigger shutdown
|
||||
Shutdown(context.Context) error //perm:admin
|
||||
|
||||
}
|
||||
|
||||
var _ storiface.WorkerCalls = *new(Worker)
|
||||
|
||||
+370
-566
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ import (
|
||||
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
|
||||
var res v0api.CommonNetStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -29,16 +29,16 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
|
||||
var res v0api.FullNodeStruct
|
||||
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
|
||||
// NewFullNodeRPCV1 creates a new http jsonrpc client.
|
||||
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
var res v1api.FullNodeStruct
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res), requestHeader, append([]jsonrpc.Option{jsonrpc.WithErrors(api.RPCErrors)}, opts...)...)
|
||||
api.GetInternalStructs(&res), requestHeader)
|
||||
|
||||
return &res, closer, err
|
||||
}
|
||||
@@ -72,7 +72,6 @@ func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.H
|
||||
api.GetInternalStructs(&res), requestHeader,
|
||||
append([]jsonrpc.Option{
|
||||
rpcenc.ReaderParamEncoder(pushUrl),
|
||||
jsonrpc.WithErrors(api.RPCErrors),
|
||||
}, opts...)...)
|
||||
|
||||
return &res, closer, err
|
||||
@@ -91,7 +90,6 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
|
||||
rpcenc.ReaderParamEncoder(pushUrl),
|
||||
jsonrpc.WithNoReconnect(),
|
||||
jsonrpc.WithTimeout(30*time.Second),
|
||||
jsonrpc.WithErrors(api.RPCErrors),
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
@@ -103,7 +101,7 @@ func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
requestHeader,
|
||||
append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
|
||||
opts...,
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
@@ -115,7 +113,7 @@ func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
requestHeader,
|
||||
append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
|
||||
opts...,
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
@@ -126,7 +124,6 @@ func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header)
|
||||
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
|
||||
api.GetInternalStructs(&res),
|
||||
requestHeader,
|
||||
jsonrpc.WithErrors(api.RPCErrors),
|
||||
)
|
||||
|
||||
return &res, closer, err
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/docgen"
|
||||
|
||||
docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
|
||||
)
|
||||
|
||||
|
||||
@@ -8,11 +8,10 @@ import (
|
||||
|
||||
"github.com/alecthomas/jsonschema"
|
||||
go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
|
||||
"github.com/ipfs/go-cid"
|
||||
meta_schema "github.com/open-rpc/meta-schema"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/docgen"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/ipfs/go-cid"
|
||||
meta_schema "github.com/open-rpc/meta-schema"
|
||||
)
|
||||
|
||||
// schemaDictEntry represents a type association passed to the jsonschema reflector.
|
||||
|
||||
+39
-167
@@ -1,38 +1,37 @@
|
||||
package docgen
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/google/uuid"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-graphsync"
|
||||
textselector "github.com/ipld/go-ipld-selector-text-lite"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
"github.com/libp2p/go-libp2p/core/metrics"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/protocol"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
"github.com/filecoin-project/go-fil-markets/filestore"
|
||||
"github.com/filecoin-project/go-multistore"
|
||||
"github.com/google/uuid"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-filestore"
|
||||
"github.com/libp2p/go-libp2p-core/metrics"
|
||||
"github.com/libp2p/go-libp2p-core/network"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
"github.com/libp2p/go-libp2p-core/protocol"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
|
||||
@@ -41,12 +40,11 @@ import (
|
||||
"github.com/filecoin-project/lotus/api/v0api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/types/ethtypes"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
|
||||
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
|
||||
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
sealing "github.com/filecoin-project/lotus/storage/pipeline"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
)
|
||||
|
||||
var ExampleValues = map[reflect.Type]interface{}{
|
||||
@@ -69,7 +67,6 @@ func init() {
|
||||
}
|
||||
|
||||
ExampleValues[reflect.TypeOf(c)] = c
|
||||
ExampleValues[reflect.TypeOf(&c)] = &c
|
||||
|
||||
c2, err := cid.Decode("bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve")
|
||||
if err != nil {
|
||||
@@ -86,7 +83,6 @@ func init() {
|
||||
}
|
||||
|
||||
ExampleValues[reflect.TypeOf(addr)] = addr
|
||||
ExampleValues[reflect.TypeOf(&addr)] = &addr
|
||||
|
||||
pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
|
||||
if err != nil {
|
||||
@@ -95,13 +91,8 @@ func init() {
|
||||
addExample(pid)
|
||||
addExample(&pid)
|
||||
|
||||
storeIDExample := imports.ID(50)
|
||||
textSelExample := textselector.Expression("Links/21/Hash/Links/42/Hash")
|
||||
apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
|
||||
clientEvent := retrievalmarket.ClientEventDealAccepted
|
||||
|
||||
block := blocks.Block(&blocks.BasicBlock{})
|
||||
ExampleValues[reflect.TypeOf(&block).Elem()] = block
|
||||
multistoreIDExample := multistore.StoreID(50)
|
||||
storeIDExample := importmgr.ImportID(50)
|
||||
|
||||
addExample(bitfield.NewFromSet([]uint64{5}))
|
||||
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
|
||||
@@ -120,6 +111,7 @@ func init() {
|
||||
addExample(abi.UnpaddedPieceSize(1024))
|
||||
addExample(abi.UnpaddedPieceSize(1024).Padded())
|
||||
addExample(abi.DealID(5432))
|
||||
addExample(filestore.StatusFileChanged)
|
||||
addExample(abi.SectorNumber(9))
|
||||
addExample(abi.SectorSize(32 * 1024 * 1024 * 1024))
|
||||
addExample(api.MpoolChange(0))
|
||||
@@ -129,32 +121,21 @@ func init() {
|
||||
addExample(api.FullAPIVersion1)
|
||||
addExample(api.PCHInbound)
|
||||
addExample(time.Minute)
|
||||
addExample(graphsync.NewRequestID())
|
||||
addExample(datatransfer.TransferID(3))
|
||||
addExample(datatransfer.Ongoing)
|
||||
addExample(storeIDExample)
|
||||
addExample(&storeIDExample)
|
||||
addExample(clientEvent)
|
||||
addExample(&clientEvent)
|
||||
addExample(multistoreIDExample)
|
||||
addExample(&multistoreIDExample)
|
||||
addExample(retrievalmarket.ClientEventDealAccepted)
|
||||
addExample(retrievalmarket.DealStatusNew)
|
||||
addExample(&textSelExample)
|
||||
addExample(&apiSelExample)
|
||||
addExample(network.ReachabilityPublic)
|
||||
addExample(build.TestNetworkVersion)
|
||||
allocationId := verifreg.AllocationId(0)
|
||||
addExample(allocationId)
|
||||
addExample(&allocationId)
|
||||
addExample(map[verifreg.AllocationId]verifreg.Allocation{})
|
||||
claimId := verifreg.ClaimId(0)
|
||||
addExample(claimId)
|
||||
addExample(&claimId)
|
||||
addExample(map[verifreg.ClaimId]verifreg.Claim{})
|
||||
addExample(build.NewestNetworkVersion)
|
||||
addExample(map[string]int{"name": 42})
|
||||
addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
|
||||
addExample(&types.ExecutionTrace{
|
||||
Msg: ExampleValue("init", reflect.TypeOf(types.MessageTrace{}), nil).(types.MessageTrace),
|
||||
MsgRct: ExampleValue("init", reflect.TypeOf(types.ReturnTrace{}), nil).(types.ReturnTrace),
|
||||
Msg: ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
|
||||
MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
|
||||
})
|
||||
addExample(map[string]types.Actor{
|
||||
"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
|
||||
@@ -162,10 +143,6 @@ func init() {
|
||||
addExample(map[string]api.MarketDeal{
|
||||
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
|
||||
})
|
||||
addExample(map[string]*api.MarketDeal{
|
||||
"t026363": ExampleValue("init", reflect.TypeOf(&api.MarketDeal{}), nil).(*api.MarketDeal),
|
||||
})
|
||||
|
||||
addExample(map[string]api.MarketBalance{
|
||||
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
|
||||
})
|
||||
@@ -203,12 +180,11 @@ func init() {
|
||||
ExampleValues[reflect.TypeOf(struct{ A multiaddr.Multiaddr }{}).Field(0).Type] = maddr
|
||||
|
||||
// miner specific
|
||||
addExample(filestore.Path(".lotusminer/fstmp123"))
|
||||
addExample(filestore2.Path(".lotusminer/fstmp123"))
|
||||
si := uint64(12)
|
||||
addExample(&si)
|
||||
addExample(retrievalmarket.DealID(5))
|
||||
addExample(abi.ActorID(1000))
|
||||
addExample(map[string]cid.Cid{})
|
||||
addExample(map[string][]api.SealedRef{
|
||||
"98000": {
|
||||
api.SealedRef{
|
||||
@@ -219,10 +195,10 @@ func init() {
|
||||
},
|
||||
})
|
||||
addExample(api.SectorState(sealing.Proving))
|
||||
addExample(storiface.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
|
||||
addExample(stores.ID("76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8"))
|
||||
addExample(storiface.FTUnsealed)
|
||||
addExample(storiface.PathSealing)
|
||||
addExample(map[storiface.ID][]storiface.Decl{
|
||||
addExample(map[stores.ID][]stores.Decl{
|
||||
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": {
|
||||
{
|
||||
SectorID: abi.SectorID{Miner: 1000, Number: 100},
|
||||
@@ -230,7 +206,7 @@ func init() {
|
||||
},
|
||||
},
|
||||
})
|
||||
addExample(map[storiface.ID]string{
|
||||
addExample(map[stores.ID]string{
|
||||
"76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path",
|
||||
})
|
||||
addExample(map[uuid.UUID][]storiface.WorkerJob{
|
||||
@@ -254,18 +230,16 @@ func init() {
|
||||
Hostname: "host",
|
||||
Resources: storiface.WorkerResources{
|
||||
MemPhysical: 256 << 30,
|
||||
MemUsed: 2 << 30,
|
||||
MemSwap: 120 << 30,
|
||||
MemSwapUsed: 2 << 30,
|
||||
MemReserved: 2 << 30,
|
||||
CPUs: 64,
|
||||
GPUs: []string{"aGPU 1337"},
|
||||
Resources: storiface.ResourceTable,
|
||||
},
|
||||
},
|
||||
Enabled: true,
|
||||
MemUsedMin: 0,
|
||||
MemUsedMax: 0,
|
||||
GpuUsed: 0,
|
||||
GpuUsed: false,
|
||||
CpuUse: 0,
|
||||
},
|
||||
})
|
||||
@@ -273,20 +247,10 @@ func init() {
|
||||
addExample(map[abi.SectorNumber]string{
|
||||
123: "can't acquire read lock",
|
||||
})
|
||||
addExample(json.RawMessage(`"json raw message"`))
|
||||
addExample(map[api.SectorState]int{
|
||||
api.SectorState(sealing.Proving): 120,
|
||||
})
|
||||
addExample([]abi.SectorNumber{123, 124})
|
||||
addExample([]storiface.SectorLock{
|
||||
{
|
||||
Sector: abi.SectorID{Number: 123, Miner: 1000},
|
||||
Write: [storiface.FileTypes]uint{0, 0, 1},
|
||||
Read: [storiface.FileTypes]uint{2, 3, 0},
|
||||
},
|
||||
})
|
||||
storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
|
||||
addExample(&storifaceid)
|
||||
|
||||
// worker specific
|
||||
addExample(storiface.AcquireMove)
|
||||
@@ -301,8 +265,7 @@ func init() {
|
||||
"title": "Lotus RPC API",
|
||||
"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
|
||||
},
|
||||
"methods": []interface{}{},
|
||||
},
|
||||
"methods": []interface{}{}},
|
||||
)
|
||||
|
||||
addExample(api.CheckStatusCode(0))
|
||||
@@ -322,93 +285,10 @@ func init() {
|
||||
State: "ShardStateAvailable",
|
||||
Error: "<error>",
|
||||
})
|
||||
addExample(storiface.ResourceTable)
|
||||
addExample(network.ScopeStat{
|
||||
Memory: 123,
|
||||
NumStreamsInbound: 1,
|
||||
NumStreamsOutbound: 2,
|
||||
NumConnsInbound: 3,
|
||||
NumConnsOutbound: 4,
|
||||
NumFD: 5,
|
||||
})
|
||||
addExample(map[string]network.ScopeStat{
|
||||
"abc": {
|
||||
Memory: 123,
|
||||
NumStreamsInbound: 1,
|
||||
NumStreamsOutbound: 2,
|
||||
NumConnsInbound: 3,
|
||||
NumConnsOutbound: 4,
|
||||
NumFD: 5,
|
||||
},
|
||||
})
|
||||
addExample(api.NetLimit{
|
||||
Memory: 123,
|
||||
StreamsInbound: 1,
|
||||
StreamsOutbound: 2,
|
||||
Streams: 3,
|
||||
ConnsInbound: 3,
|
||||
ConnsOutbound: 4,
|
||||
Conns: 4,
|
||||
FD: 5,
|
||||
})
|
||||
|
||||
addExample(map[string]bitfield.BitField{
|
||||
"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
|
||||
})
|
||||
addExample(&api.RaftStateData{
|
||||
NonceMap: make(map[address.Address]uint64),
|
||||
MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
|
||||
})
|
||||
|
||||
addExample(http.Header{
|
||||
"Authorization": []string{"Bearer ey.."},
|
||||
})
|
||||
|
||||
addExample(map[storiface.SectorFileType]storiface.SectorLocation{
|
||||
storiface.FTSealed: {
|
||||
Local: false,
|
||||
URL: "https://example.com/sealingservice/sectors/s-f0123-12345",
|
||||
Headers: nil,
|
||||
},
|
||||
})
|
||||
|
||||
ethint := ethtypes.EthUint64(5)
|
||||
addExample(ethint)
|
||||
addExample(ðint)
|
||||
|
||||
ethaddr, _ := ethtypes.ParseEthAddress("0x5CbEeCF99d3fDB3f25E309Cc264f240bb0664031")
|
||||
addExample(ethaddr)
|
||||
addExample(ðaddr)
|
||||
|
||||
ethhash, _ := ethtypes.EthHashFromCid(c)
|
||||
addExample(ethhash)
|
||||
addExample(ðhash)
|
||||
|
||||
ethFeeHistoryReward := [][]ethtypes.EthBigInt{}
|
||||
addExample(ðFeeHistoryReward)
|
||||
|
||||
addExample(&uuid.UUID{})
|
||||
|
||||
filterid := ethtypes.EthFilterID(ethhash)
|
||||
addExample(filterid)
|
||||
addExample(&filterid)
|
||||
|
||||
subid := ethtypes.EthSubscriptionID(ethhash)
|
||||
addExample(subid)
|
||||
addExample(&subid)
|
||||
|
||||
pstring := func(s string) *string { return &s }
|
||||
addExample(ðtypes.EthFilterSpec{
|
||||
FromBlock: pstring("2301220"),
|
||||
Address: []ethtypes.EthAddress{ethaddr},
|
||||
})
|
||||
|
||||
percent := types.Percent(123)
|
||||
addExample(percent)
|
||||
addExample(&percent)
|
||||
}
|
||||
|
||||
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
|
||||
|
||||
switch pkg {
|
||||
case "api": // latest
|
||||
switch name {
|
||||
@@ -428,10 +308,6 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
|
||||
i = &api.WorkerStruct{}
|
||||
t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
|
||||
case "Gateway":
|
||||
i = &api.GatewayStruct{}
|
||||
t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
|
||||
permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
|
||||
default:
|
||||
panic("unknown type")
|
||||
}
|
||||
@@ -459,7 +335,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
|
||||
switch t.Kind() {
|
||||
case reflect.Slice:
|
||||
out := reflect.New(t).Elem()
|
||||
out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
|
||||
reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
|
||||
return out.Interface()
|
||||
case reflect.Chan:
|
||||
return ExampleValue(method, t.Elem(), nil)
|
||||
@@ -478,11 +354,8 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
|
||||
case reflect.Ptr:
|
||||
if t.Elem().Kind() == reflect.Struct {
|
||||
es := exampleStruct(method, t.Elem(), t)
|
||||
ExampleValues[t] = es
|
||||
//ExampleValues[t] = es
|
||||
return es
|
||||
} else if t.Elem().Kind() == reflect.String {
|
||||
str := "string value"
|
||||
return &str
|
||||
}
|
||||
case reflect.Interface:
|
||||
return struct{}{}
|
||||
@@ -498,8 +371,7 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
|
||||
if f.Type == parent {
|
||||
continue
|
||||
}
|
||||
|
||||
if f.IsExported() {
|
||||
if strings.Title(f.Name) == f.Name {
|
||||
ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
package api
|
||||
|
||||
import apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
|
||||
func CreateEthRPCAliases(as apitypes.Aliaser) {
|
||||
// TODO: maybe use reflect to automatically register all the eth aliases
|
||||
as.AliasMethod("eth_accounts", "Filecoin.EthAccounts")
|
||||
as.AliasMethod("eth_blockNumber", "Filecoin.EthBlockNumber")
|
||||
as.AliasMethod("eth_getBlockTransactionCountByNumber", "Filecoin.EthGetBlockTransactionCountByNumber")
|
||||
as.AliasMethod("eth_getBlockTransactionCountByHash", "Filecoin.EthGetBlockTransactionCountByHash")
|
||||
|
||||
as.AliasMethod("eth_getBlockByHash", "Filecoin.EthGetBlockByHash")
|
||||
as.AliasMethod("eth_getBlockByNumber", "Filecoin.EthGetBlockByNumber")
|
||||
as.AliasMethod("eth_getTransactionByHash", "Filecoin.EthGetTransactionByHash")
|
||||
as.AliasMethod("eth_getTransactionCount", "Filecoin.EthGetTransactionCount")
|
||||
as.AliasMethod("eth_getTransactionReceipt", "Filecoin.EthGetTransactionReceipt")
|
||||
as.AliasMethod("eth_getTransactionByBlockHashAndIndex", "Filecoin.EthGetTransactionByBlockHashAndIndex")
|
||||
as.AliasMethod("eth_getTransactionByBlockNumberAndIndex", "Filecoin.EthGetTransactionByBlockNumberAndIndex")
|
||||
|
||||
as.AliasMethod("eth_getCode", "Filecoin.EthGetCode")
|
||||
as.AliasMethod("eth_getStorageAt", "Filecoin.EthGetStorageAt")
|
||||
as.AliasMethod("eth_getBalance", "Filecoin.EthGetBalance")
|
||||
as.AliasMethod("eth_chainId", "Filecoin.EthChainId")
|
||||
as.AliasMethod("eth_syncing", "Filecoin.EthSyncing")
|
||||
as.AliasMethod("eth_feeHistory", "Filecoin.EthFeeHistory")
|
||||
as.AliasMethod("eth_protocolVersion", "Filecoin.EthProtocolVersion")
|
||||
as.AliasMethod("eth_maxPriorityFeePerGas", "Filecoin.EthMaxPriorityFeePerGas")
|
||||
as.AliasMethod("eth_gasPrice", "Filecoin.EthGasPrice")
|
||||
as.AliasMethod("eth_sendRawTransaction", "Filecoin.EthSendRawTransaction")
|
||||
as.AliasMethod("eth_estimateGas", "Filecoin.EthEstimateGas")
|
||||
as.AliasMethod("eth_call", "Filecoin.EthCall")
|
||||
|
||||
as.AliasMethod("eth_getLogs", "Filecoin.EthGetLogs")
|
||||
as.AliasMethod("eth_getFilterChanges", "Filecoin.EthGetFilterChanges")
|
||||
as.AliasMethod("eth_getFilterLogs", "Filecoin.EthGetFilterLogs")
|
||||
as.AliasMethod("eth_newFilter", "Filecoin.EthNewFilter")
|
||||
as.AliasMethod("eth_newBlockFilter", "Filecoin.EthNewBlockFilter")
|
||||
as.AliasMethod("eth_newPendingTransactionFilter", "Filecoin.EthNewPendingTransactionFilter")
|
||||
as.AliasMethod("eth_uninstallFilter", "Filecoin.EthUninstallFilter")
|
||||
as.AliasMethod("eth_subscribe", "Filecoin.EthSubscribe")
|
||||
as.AliasMethod("eth_unsubscribe", "Filecoin.EthUnsubscribe")
|
||||
|
||||
as.AliasMethod("trace_block", "Filecoin.EthTraceBlock")
|
||||
as.AliasMethod("trace_replayBlockTransactions", "Filecoin.EthTraceReplayBlockTransactions")
|
||||
|
||||
as.AliasMethod("debug_traceCall", "Filecoin.EthDebugTraceCall")
|
||||
as.AliasMethod("debug_traceBlockByHash", "Filecoin.EthDebugTraceBlockByHash")
|
||||
as.AliasMethod("debug_traceBlockByNumber", "Filecoin.EthDebugTraceBlockByNumber")
|
||||
|
||||
as.AliasMethod("net_version", "Filecoin.NetVersion")
|
||||
as.AliasMethod("net_listening", "Filecoin.NetListening")
|
||||
|
||||
as.AliasMethod("web3_clientVersion", "Filecoin.Web3ClientVersion")
|
||||
}
|
||||
+101
-1300
File diff suppressed because it is too large
Load Diff
+563
-3057
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,3 @@
|
||||
// stm: #unit
|
||||
package api
|
||||
|
||||
import (
|
||||
@@ -30,7 +29,6 @@ type StrC struct {
|
||||
}
|
||||
|
||||
func TestGetInternalStructs(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
|
||||
var proxy StrA
|
||||
|
||||
sts := GetInternalStructs(&proxy)
|
||||
@@ -46,7 +44,6 @@ func TestGetInternalStructs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNestedInternalStructs(t *testing.T) {
|
||||
//stm: @OTHER_IMPLEMENTATION_API_STRUCTS_001
|
||||
var proxy StrC
|
||||
|
||||
// check that only the top-level internal struct gets picked up
|
||||
|
||||
+19
-241
@@ -5,26 +5,20 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-graphsync"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
"github.com/ipld/go-ipld-prime/codec/dagjson"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
ma "github.com/multiformats/go-multiaddr"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
pubsub "github.com/libp2p/go-libp2p-pubsub"
|
||||
ma "github.com/multiformats/go-multiaddr"
|
||||
)
|
||||
|
||||
// TODO: check if this exists anywhere else
|
||||
|
||||
type MultiaddrSlice []ma.Multiaddr
|
||||
|
||||
func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
|
||||
@@ -56,46 +50,8 @@ type PubsubScore struct {
|
||||
Score *pubsub.PeerScoreSnapshot
|
||||
}
|
||||
|
||||
// MessageSendSpec contains optional fields which modify message sending behavior
|
||||
type MessageSendSpec struct {
|
||||
// MaxFee specifies a cap on network fees related to this message
|
||||
MaxFee abi.TokenAmount
|
||||
|
||||
// MsgUuid specifies a unique message identifier which can be used on node (or node cluster)
|
||||
// level to prevent double-sends of messages even when nonce generation is not handled by sender
|
||||
MsgUuid uuid.UUID
|
||||
|
||||
// MaximizeFeeCap makes message FeeCap be based entirely on MaxFee
|
||||
MaximizeFeeCap bool
|
||||
}
|
||||
|
||||
type MpoolMessageWhole struct {
|
||||
Msg *types.Message
|
||||
Spec *MessageSendSpec
|
||||
}
|
||||
|
||||
// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
|
||||
type GraphSyncDataTransfer struct {
|
||||
// GraphSync request id for this transfer
|
||||
RequestID *graphsync.RequestID
|
||||
// Graphsync state for this transfer
|
||||
RequestState string
|
||||
// If a channel ID is present, indicates whether this is the current graphsync request for this channel
|
||||
// (could have changed in a restart)
|
||||
IsCurrentChannelRequest bool
|
||||
// Data transfer channel ID for this transfer
|
||||
ChannelID *datatransfer.ChannelID
|
||||
// Data transfer state for this transfer
|
||||
ChannelState *DataTransferChannel
|
||||
// Diagnostic information about this request -- and unexpected inconsistencies in
|
||||
// request state
|
||||
Diagnostics []string
|
||||
}
|
||||
|
||||
// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
|
||||
type TransferDiagnostics struct {
|
||||
ReceivingTransfers []*GraphSyncDataTransfer
|
||||
SendingTransfers []*GraphSyncDataTransfer
|
||||
}
|
||||
|
||||
type DataTransferChannel struct {
|
||||
@@ -120,12 +76,16 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
|
||||
IsSender: channelState.Sender() == hostID,
|
||||
Message: channelState.Message(),
|
||||
}
|
||||
voucher := channelState.Voucher()
|
||||
voucherJSON, err := ipld.Encode(voucher.Voucher, dagjson.Encode)
|
||||
if err != nil {
|
||||
channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
|
||||
stringer, ok := channelState.Voucher().(fmt.Stringer)
|
||||
if ok {
|
||||
channel.Voucher = stringer.String()
|
||||
} else {
|
||||
channel.Voucher = string(voucherJSON)
|
||||
voucherJSON, err := json.Marshal(channelState.Voucher())
|
||||
if err != nil {
|
||||
channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
|
||||
} else {
|
||||
channel.Voucher = string(voucherJSON)
|
||||
}
|
||||
}
|
||||
if channel.IsSender {
|
||||
channel.IsInitiator = !channelState.IsPull()
|
||||
@@ -139,22 +99,6 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
|
||||
return channel
|
||||
}
|
||||
|
||||
type NetStat struct {
|
||||
System *network.ScopeStat `json:",omitempty"`
|
||||
Transient *network.ScopeStat `json:",omitempty"`
|
||||
Services map[string]network.ScopeStat `json:",omitempty"`
|
||||
Protocols map[string]network.ScopeStat `json:",omitempty"`
|
||||
Peers map[string]network.ScopeStat `json:",omitempty"`
|
||||
}
|
||||
|
||||
type NetLimit struct {
|
||||
Memory int64 `json:",omitempty"`
|
||||
|
||||
Streams, StreamsInbound, StreamsOutbound int
|
||||
Conns, ConnsInbound, ConnsOutbound int
|
||||
FD int
|
||||
}
|
||||
|
||||
type NetBlockList struct {
|
||||
Peers []peer.ID
|
||||
IPAddrs []string
|
||||
@@ -250,170 +194,4 @@ type RetrievalInfo struct {
|
||||
|
||||
TransferChannelID *datatransfer.ChannelID
|
||||
DataTransfer *DataTransferChannel
|
||||
|
||||
// optional event if part of ClientGetRetrievalUpdates
|
||||
Event *retrievalmarket.ClientEvent
|
||||
}
|
||||
|
||||
type RestrievalRes struct {
|
||||
DealID retrievalmarket.DealID
|
||||
}
|
||||
|
||||
// Selector specifies ipld selector string
|
||||
// - if the string starts with '{', it's interpreted as json selector string
|
||||
// see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
|
||||
// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
|
||||
// see https://github.com/ipld/go-ipld-selector-text-lite
|
||||
type Selector string
|
||||
|
||||
type DagSpec struct {
|
||||
// DataSelector matches data to be retrieved
|
||||
// - when using textselector, the path specifies subtree
|
||||
// - the matched graph must have a single root
|
||||
DataSelector *Selector
|
||||
|
||||
// ExportMerkleProof is applicable only when exporting to a CAR file via a path textselector
|
||||
// When true, in addition to the selection target, the resulting CAR will contain every block along the
|
||||
// path back to, and including the original root
|
||||
// When false the resulting CAR contains only the blocks of the target subdag
|
||||
ExportMerkleProof bool
|
||||
}
|
||||
|
||||
type ExportRef struct {
|
||||
Root cid.Cid
|
||||
|
||||
// DAGs array specifies a list of DAGs to export
|
||||
// - If exporting into unixfs files, only one DAG is supported, DataSelector is only used to find the targeted root node
|
||||
// - If exporting into a car file
|
||||
// - When exactly one text-path DataSelector is specified exports the subgraph and its full merkle-path from the original root
|
||||
// - Otherwise ( multiple paths and/or JSON selector specs) determines each individual subroot and exports the subtrees as a multi-root car
|
||||
// - When not specified defaults to a single DAG:
|
||||
// - Data - the entire DAG: `{"R":{"l":{"none":{}},":>":{"a":{">":{"@":{}}}}}}`
|
||||
DAGs []DagSpec
|
||||
|
||||
FromLocalCAR string // if specified, get data from a local CARv2 file.
|
||||
DealID retrievalmarket.DealID
|
||||
}
|
||||
|
||||
type MinerInfo struct {
|
||||
Owner address.Address // Must be an ID-address.
|
||||
Worker address.Address // Must be an ID-address.
|
||||
NewWorker address.Address // Must be an ID-address.
|
||||
ControlAddresses []address.Address // Must be an ID-addresses.
|
||||
WorkerChangeEpoch abi.ChainEpoch
|
||||
PeerId *peer.ID
|
||||
Multiaddrs []abi.Multiaddrs
|
||||
WindowPoStProofType abi.RegisteredPoStProof
|
||||
SectorSize abi.SectorSize
|
||||
WindowPoStPartitionSectors uint64
|
||||
ConsensusFaultElapsed abi.ChainEpoch
|
||||
PendingOwnerAddress *address.Address
|
||||
Beneficiary address.Address
|
||||
BeneficiaryTerm *miner.BeneficiaryTerm
|
||||
PendingBeneficiaryTerm *miner.PendingBeneficiaryChange
|
||||
}
|
||||
|
||||
type NetworkParams struct {
|
||||
NetworkName dtypes.NetworkName
|
||||
BlockDelaySecs uint64
|
||||
ConsensusMinerMinPower abi.StoragePower
|
||||
SupportedProofTypes []abi.RegisteredSealProof
|
||||
PreCommitChallengeDelay abi.ChainEpoch
|
||||
ForkUpgradeParams ForkUpgradeParams
|
||||
Eip155ChainID int
|
||||
}
|
||||
|
||||
type ForkUpgradeParams struct {
|
||||
UpgradeSmokeHeight abi.ChainEpoch
|
||||
UpgradeBreezeHeight abi.ChainEpoch
|
||||
UpgradeIgnitionHeight abi.ChainEpoch
|
||||
UpgradeLiftoffHeight abi.ChainEpoch
|
||||
UpgradeAssemblyHeight abi.ChainEpoch
|
||||
UpgradeRefuelHeight abi.ChainEpoch
|
||||
UpgradeTapeHeight abi.ChainEpoch
|
||||
UpgradeKumquatHeight abi.ChainEpoch
|
||||
BreezeGasTampingDuration abi.ChainEpoch
|
||||
UpgradeCalicoHeight abi.ChainEpoch
|
||||
UpgradePersianHeight abi.ChainEpoch
|
||||
UpgradeOrangeHeight abi.ChainEpoch
|
||||
UpgradeClausHeight abi.ChainEpoch
|
||||
UpgradeTrustHeight abi.ChainEpoch
|
||||
UpgradeNorwegianHeight abi.ChainEpoch
|
||||
UpgradeTurboHeight abi.ChainEpoch
|
||||
UpgradeHyperdriveHeight abi.ChainEpoch
|
||||
UpgradeChocolateHeight abi.ChainEpoch
|
||||
UpgradeOhSnapHeight abi.ChainEpoch
|
||||
UpgradeSkyrHeight abi.ChainEpoch
|
||||
UpgradeSharkHeight abi.ChainEpoch
|
||||
UpgradeHyggeHeight abi.ChainEpoch
|
||||
UpgradeLightningHeight abi.ChainEpoch
|
||||
UpgradeThunderHeight abi.ChainEpoch
|
||||
UpgradeWatermelonHeight abi.ChainEpoch
|
||||
}
|
||||
|
||||
type NonceMapType map[address.Address]uint64
|
||||
type MsgUuidMapType map[uuid.UUID]*types.SignedMessage
|
||||
|
||||
type RaftStateData struct {
|
||||
NonceMap NonceMapType
|
||||
MsgUuids MsgUuidMapType
|
||||
}
|
||||
|
||||
func (n *NonceMapType) MarshalJSON() ([]byte, error) {
|
||||
marshalled := make(map[string]uint64)
|
||||
for a, n := range *n {
|
||||
marshalled[a.String()] = n
|
||||
}
|
||||
return json.Marshal(marshalled)
|
||||
}
|
||||
|
||||
func (n *NonceMapType) UnmarshalJSON(b []byte) error {
|
||||
unmarshalled := make(map[string]uint64)
|
||||
err := json.Unmarshal(b, &unmarshalled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*n = make(map[address.Address]uint64)
|
||||
for saddr, nonce := range unmarshalled {
|
||||
a, err := address.NewFromString(saddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
(*n)[a] = nonce
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MsgUuidMapType) MarshalJSON() ([]byte, error) {
|
||||
marshalled := make(map[string]*types.SignedMessage)
|
||||
for u, msg := range *m {
|
||||
marshalled[u.String()] = msg
|
||||
}
|
||||
return json.Marshal(marshalled)
|
||||
}
|
||||
|
||||
func (m *MsgUuidMapType) UnmarshalJSON(b []byte) error {
|
||||
unmarshalled := make(map[string]*types.SignedMessage)
|
||||
err := json.Unmarshal(b, &unmarshalled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*m = make(map[uuid.UUID]*types.SignedMessage)
|
||||
for suid, msg := range unmarshalled {
|
||||
u, err := uuid.Parse(suid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
(*m)[u] = msg
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChainExportConfig holds configuration for chain ranged exports.
|
||||
type ChainExportConfig struct {
|
||||
WriteBufferSize int
|
||||
NumWorkers int
|
||||
IncludeMessages bool
|
||||
IncludeReceipts bool
|
||||
IncludeStateRoots bool
|
||||
}
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
package apitypes
|
||||
|
||||
type Aliaser interface {
|
||||
AliasMethod(alias, original string)
|
||||
}
|
||||
+20
-84
@@ -3,30 +3,25 @@ package v0api
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
textselector "github.com/ipld/go-ipld-selector-text-lite"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
|
||||
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
abinetwork "github.com/filecoin-project/go-state-types/network"
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p-core/peer"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
"github.com/filecoin-project/lotus/node/repo/imports"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/golang/mock/mockgen -destination=v0mocks/mock_full.go -package=v0mocks . FullNode
|
||||
@@ -113,9 +108,6 @@ type FullNode interface {
|
||||
// ChainDeleteObj deletes node referenced by the given CID
|
||||
ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
|
||||
|
||||
// ChainPutObj puts and object into the blockstore
|
||||
ChainPutObj(context.Context, blocks.Block) error
|
||||
|
||||
// ChainHasObj checks if a given CID exists in the chain blockstore.
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
|
||||
|
||||
@@ -140,7 +132,7 @@ type FullNode interface {
|
||||
|
||||
// ChainGetPath returns a set of revert/apply operations needed to get from
|
||||
// one tipset to another, for example:
|
||||
// ```
|
||||
//```
|
||||
// to
|
||||
// ^
|
||||
// from tAA
|
||||
@@ -149,7 +141,7 @@ type FullNode interface {
|
||||
// ^---*--^
|
||||
// ^
|
||||
// tRR
|
||||
// ```
|
||||
//```
|
||||
// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
|
||||
ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) //perm:read
|
||||
|
||||
@@ -313,7 +305,7 @@ type FullNode interface {
|
||||
// ClientImport imports file under the specified path into filestore.
|
||||
ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
|
||||
// ClientRemoveImport removes file import
|
||||
ClientRemoveImport(ctx context.Context, importID imports.ID) error //perm:admin
|
||||
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
|
||||
// ClientStartDeal proposes a deal with a miner.
|
||||
ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
|
||||
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
|
||||
@@ -333,10 +325,10 @@ type FullNode interface {
|
||||
// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
|
||||
ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (api.QueryOffer, error) //perm:read
|
||||
// ClientRetrieve initiates the retrieval of a file, as specified in the order.
|
||||
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error //perm:admin
|
||||
ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error //perm:admin
|
||||
// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
|
||||
// of status updates.
|
||||
ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
|
||||
ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
|
||||
// ClientQueryAsk returns a signed StorageAsk from the specified miner.
|
||||
// ClientListRetrievals returns information about retrievals made by the local client
|
||||
ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) //perm:write
|
||||
@@ -366,12 +358,12 @@ type FullNode interface {
|
||||
ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
|
||||
|
||||
// ClientUnimport removes references to the specified file from filestore
|
||||
// ClientUnimport(path string)
|
||||
//ClientUnimport(path string)
|
||||
|
||||
// ClientListImports lists imported files and their root CIDs
|
||||
ClientListImports(ctx context.Context) ([]api.Import, error) //perm:write
|
||||
|
||||
// ClientListAsks() []Ask
|
||||
//ClientListAsks() []Ask
|
||||
|
||||
// MethodGroup: State
|
||||
// The State methods are used to query, inspect, and interact with chain state.
|
||||
@@ -386,7 +378,7 @@ type FullNode interface {
|
||||
StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
|
||||
// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
|
||||
//
|
||||
// If a tipset key is provided, and a replacing message is not found on chain,
|
||||
// If a tipset key is provided, and a replacing message is found on chain,
|
||||
// the method will return an error saying that the message wasn't found
|
||||
//
|
||||
// If no tipset key is provided, the appropriate tipset is looked up, and if
|
||||
@@ -423,7 +415,7 @@ type FullNode interface {
|
||||
// StateMinerPower returns the power of the indicated miner
|
||||
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
|
||||
// StateMinerInfo returns info about the indicated miner
|
||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) //perm:read
|
||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
|
||||
// StateMinerDeadlines returns all the proving deadlines for the given miner
|
||||
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
|
||||
// StateMinerPartitions returns all partitions in the specified deadline
|
||||
@@ -528,19 +520,9 @@ type FullNode interface {
|
||||
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
|
||||
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
|
||||
// StateMarketDeals returns information about every deal in the Storage Market
|
||||
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*api.MarketDeal, error) //perm:read
|
||||
StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
|
||||
// StateMarketStorageDeal returns information about the indicated deal
|
||||
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
|
||||
// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal.
|
||||
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetAllocation returns the allocation for a given address and allocation ID.
|
||||
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetAllocations returns the all the allocations for a given client.
|
||||
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
|
||||
// StateGetClaim returns the claim for a given address and claim ID.
|
||||
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
|
||||
// StateGetClaims returns the all the claims for a given provider.
|
||||
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
|
||||
// StateLookupID retrieves the ID address of the given address
|
||||
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateAccountKey returns the public key address of the given ID address
|
||||
@@ -601,7 +583,7 @@ type FullNode interface {
|
||||
// Returns nil if there is no entry in the data cap table for the
|
||||
// address.
|
||||
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
|
||||
// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
|
||||
// StateVerifiedClientStatus returns the address of the Verified Registry's root key
|
||||
StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
|
||||
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
|
||||
// can issue. It takes the deal size and verified status as parameters.
|
||||
@@ -615,18 +597,6 @@ type FullNode interface {
|
||||
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
|
||||
// StateNetworkVersion returns the network version at the given tipset
|
||||
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
|
||||
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
|
||||
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
|
||||
// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
|
||||
StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
|
||||
|
||||
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
|
||||
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
|
||||
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
|
||||
|
||||
// StateGetNetworkParams return current network params
|
||||
StateGetNetworkParams(ctx context.Context) (*api.NetworkParams, error) //perm:read
|
||||
|
||||
// MethodGroup: Msig
|
||||
// The Msig methods are used to interact with multisig wallets on the
|
||||
@@ -640,14 +610,14 @@ type FullNode interface {
|
||||
// It takes the following params: <multisig address>, <start epoch>, <end epoch>
|
||||
MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
|
||||
|
||||
// MsigGetPending returns pending transactions for the given multisig
|
||||
// wallet. Once pending transactions are fully approved, they will no longer
|
||||
// appear here.
|
||||
//MsigGetPending returns pending transactions for the given multisig
|
||||
//wallet. Once pending transactions are fully approved, they will no longer
|
||||
//appear here.
|
||||
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) //perm:read
|
||||
|
||||
// MsigCreate creates a multisig wallet
|
||||
// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
|
||||
// <initial balance>, <sender address of the create msg>, <gas price>
|
||||
//<initial balance>, <sender address of the create msg>, <gas price>
|
||||
MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
|
||||
// MsigPropose proposes a multisig message
|
||||
// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
|
||||
@@ -739,37 +709,3 @@ type FullNode interface {
|
||||
// the path specified when calling CreateBackup is within the base path
|
||||
CreateBackup(ctx context.Context, fpath string) error //perm:admin
|
||||
}
|
||||
|
||||
func OfferOrder(o api.QueryOffer, client address.Address) RetrievalOrder {
|
||||
return RetrievalOrder{
|
||||
Root: o.Root,
|
||||
Piece: o.Piece,
|
||||
Size: o.Size,
|
||||
Total: o.MinPrice,
|
||||
UnsealPrice: o.UnsealPrice,
|
||||
PaymentInterval: o.PaymentInterval,
|
||||
PaymentIntervalIncrease: o.PaymentIntervalIncrease,
|
||||
Client: client,
|
||||
|
||||
Miner: o.Miner,
|
||||
MinerPeer: &o.MinerPeer,
|
||||
}
|
||||
}
|
||||
|
||||
type RetrievalOrder struct {
|
||||
// TODO: make this less unixfs specific
|
||||
Root cid.Cid
|
||||
Piece *cid.Cid
|
||||
DatamodelPathSelector *textselector.Expression
|
||||
Size uint64
|
||||
|
||||
FromLocalCAR string // if specified, get data from a local CARv2 file.
|
||||
// TODO: support offset
|
||||
Total types.BigInt
|
||||
UnsealPrice types.BigInt
|
||||
PaymentInterval uint64
|
||||
PaymentIntervalIncrease uint64
|
||||
Client address.Address
|
||||
Miner address.Address
|
||||
MinerPeer *retrievalmarket.RetrievalPeer
|
||||
}
|
||||
|
||||
+3
-22
@@ -3,19 +3,16 @@ package v0api
|
||||
import (
|
||||
"context"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
"github.com/filecoin-project/go-state-types/dline"
|
||||
abinetwork "github.com/filecoin-project/go-state-types/network"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
)
|
||||
|
||||
// MODIFYING THE API INTERFACE
|
||||
@@ -36,14 +33,7 @@ import (
|
||||
// * Generate openrpc blobs
|
||||
|
||||
type Gateway interface {
|
||||
MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
|
||||
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
|
||||
MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error)
|
||||
StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)
|
||||
GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
|
||||
StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
ChainPutObj(context.Context, blocks.Block) error
|
||||
ChainHead(ctx context.Context) (*types.TipSet, error)
|
||||
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
|
||||
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
|
||||
@@ -52,31 +42,22 @@ type Gateway interface {
|
||||
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
|
||||
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
|
||||
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
|
||||
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
|
||||
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
|
||||
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
|
||||
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error)
|
||||
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
|
||||
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
|
||||
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
|
||||
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
|
||||
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
|
||||
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
|
||||
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
|
||||
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
|
||||
StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
|
||||
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
|
||||
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
|
||||
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
|
||||
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
|
||||
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
|
||||
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
|
||||
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
|
||||
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
|
||||
StateNetworkName(context.Context) (dtypes.NetworkName, error)
|
||||
StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
|
||||
StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
|
||||
StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
|
||||
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
|
||||
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
|
||||
|
||||
@@ -2,7 +2,6 @@ package v0api
|
||||
|
||||
import (
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
)
|
||||
|
||||
|
||||
+241
-600
File diff suppressed because it is too large
Load Diff
+29
-332
@@ -7,41 +7,34 @@ package v0mocks
|
||||
import (
|
||||
context "context"
|
||||
reflect "reflect"
|
||||
time "time"
|
||||
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
uuid "github.com/google/uuid"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
metrics "github.com/libp2p/go-libp2p/core/metrics"
|
||||
network0 "github.com/libp2p/go-libp2p/core/network"
|
||||
peer "github.com/libp2p/go-libp2p/core/peer"
|
||||
protocol "github.com/libp2p/go-libp2p/core/protocol"
|
||||
|
||||
address "github.com/filecoin-project/go-address"
|
||||
bitfield "github.com/filecoin-project/go-bitfield"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
|
||||
datatransfer "github.com/filecoin-project/go-data-transfer"
|
||||
retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
auth "github.com/filecoin-project/go-jsonrpc/auth"
|
||||
abi "github.com/filecoin-project/go-state-types/abi"
|
||||
big "github.com/filecoin-project/go-state-types/big"
|
||||
paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
|
||||
miner "github.com/filecoin-project/go-state-types/builtin/v9/miner"
|
||||
verifreg "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
|
||||
crypto "github.com/filecoin-project/go-state-types/crypto"
|
||||
dline "github.com/filecoin-project/go-state-types/dline"
|
||||
network "github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
api "github.com/filecoin-project/lotus/api"
|
||||
apitypes "github.com/filecoin-project/lotus/api/types"
|
||||
v0api "github.com/filecoin-project/lotus/api/v0api"
|
||||
miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
types "github.com/filecoin-project/lotus/chain/types"
|
||||
alerting "github.com/filecoin-project/lotus/journal/alerting"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||
imports "github.com/filecoin-project/lotus/node/repo/imports"
|
||||
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
|
||||
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
uuid "github.com/google/uuid"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
metrics "github.com/libp2p/go-libp2p-core/metrics"
|
||||
network0 "github.com/libp2p/go-libp2p-core/network"
|
||||
peer "github.com/libp2p/go-libp2p-core/peer"
|
||||
protocol "github.com/libp2p/go-libp2p-core/protocol"
|
||||
)
|
||||
|
||||
// MockFullNode is a mock of FullNode interface.
|
||||
@@ -381,20 +374,6 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
|
||||
}
|
||||
|
||||
// ChainPutObj mocks base method.
|
||||
func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// ChainPutObj indicates an expected call of ChainPutObj.
|
||||
func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
|
||||
}
|
||||
|
||||
// ChainReadObj mocks base method.
|
||||
func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -752,7 +731,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
|
||||
}
|
||||
|
||||
// ClientRemoveImport mocks base method.
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 imports.ID) error {
|
||||
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
@@ -780,7 +759,7 @@ func (mr *MockFullNodeMockRecorder) ClientRestartDataTransfer(arg0, arg1, arg2,
|
||||
}
|
||||
|
||||
// ClientRetrieve mocks base method.
|
||||
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) error {
|
||||
func (m *MockFullNode) ClientRetrieve(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ClientRetrieve", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(error)
|
||||
@@ -808,7 +787,7 @@ func (mr *MockFullNodeMockRecorder) ClientRetrieveTryRestartInsufficientFunds(ar
|
||||
}
|
||||
|
||||
// ClientRetrieveWithEvents mocks base method.
|
||||
func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 v0api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
|
||||
func (m *MockFullNode) ClientRetrieveWithEvents(arg0 context.Context, arg1 api.RetrievalOrder, arg2 *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "ClientRetrieveWithEvents", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(<-chan marketevents.RetrievalEvent)
|
||||
@@ -971,21 +950,6 @@ func (mr *MockFullNodeMockRecorder) ID(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockFullNode)(nil).ID), arg0)
|
||||
}
|
||||
|
||||
// LogAlerts mocks base method.
|
||||
func (m *MockFullNode) LogAlerts(arg0 context.Context) ([]alerting.Alert, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "LogAlerts", arg0)
|
||||
ret0, _ := ret[0].([]alerting.Alert)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// LogAlerts indicates an expected call of LogAlerts.
|
||||
func (mr *MockFullNodeMockRecorder) LogAlerts(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogAlerts", reflect.TypeOf((*MockFullNode)(nil).LogAlerts), arg0)
|
||||
}
|
||||
|
||||
// LogList mocks base method.
|
||||
func (m *MockFullNode) LogList(arg0 context.Context) ([]string, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -1743,21 +1707,6 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
|
||||
}
|
||||
|
||||
// NetLimit mocks base method.
|
||||
func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
|
||||
ret0, _ := ret[0].(api.NetLimit)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// NetLimit indicates an expected call of NetLimit.
|
||||
func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
|
||||
}
|
||||
|
||||
// NetPeerInfo mocks base method.
|
||||
func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -1788,64 +1737,6 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
|
||||
}
|
||||
|
||||
// NetPing mocks base method.
|
||||
func (m *MockFullNode) NetPing(arg0 context.Context, arg1 peer.ID) (time.Duration, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetPing", arg0, arg1)
|
||||
ret0, _ := ret[0].(time.Duration)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// NetPing indicates an expected call of NetPing.
|
||||
func (mr *MockFullNodeMockRecorder) NetPing(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPing", reflect.TypeOf((*MockFullNode)(nil).NetPing), arg0, arg1)
|
||||
}
|
||||
|
||||
// NetProtectAdd mocks base method.
|
||||
func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// NetProtectAdd indicates an expected call of NetProtectAdd.
|
||||
func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
|
||||
}
|
||||
|
||||
// NetProtectList mocks base method.
|
||||
func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetProtectList", arg0)
|
||||
ret0, _ := ret[0].([]peer.ID)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// NetProtectList indicates an expected call of NetProtectList.
|
||||
func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
|
||||
}
|
||||
|
||||
// NetProtectRemove mocks base method.
|
||||
func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// NetProtectRemove indicates an expected call of NetProtectRemove.
|
||||
func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
|
||||
}
|
||||
|
||||
// NetPubsubScores mocks base method.
|
||||
func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -1861,35 +1752,6 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
|
||||
}
|
||||
|
||||
// NetSetLimit mocks base method.
|
||||
func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// NetSetLimit indicates an expected call of NetSetLimit.
|
||||
func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// NetStat mocks base method.
|
||||
func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
|
||||
ret0, _ := ret[0].(api.NetStat)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// NetStat indicates an expected call of NetStat.
|
||||
func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
|
||||
}
|
||||
|
||||
// PaychAllocateLane mocks base method.
|
||||
func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2158,21 +2020,6 @@ func (mr *MockFullNodeMockRecorder) Shutdown(arg0 interface{}) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockFullNode)(nil).Shutdown), arg0)
|
||||
}
|
||||
|
||||
// StartTime mocks base method.
|
||||
func (m *MockFullNode) StartTime(arg0 context.Context) (time.Time, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StartTime", arg0)
|
||||
ret0, _ := ret[0].(time.Time)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StartTime indicates an expected call of StartTime.
|
||||
func (mr *MockFullNodeMockRecorder) StartTime(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartTime", reflect.TypeOf((*MockFullNode)(nil).StartTime), arg0)
|
||||
}
|
||||
|
||||
// StateAccountKey mocks base method.
|
||||
func (m *MockFullNode) StateAccountKey(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (address.Address, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2188,36 +2035,6 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StateActorCodeCIDs mocks base method.
|
||||
func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
|
||||
ret0, _ := ret[0].(map[string]cid.Cid)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
|
||||
func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
|
||||
}
|
||||
|
||||
// StateActorManifestCID mocks base method.
|
||||
func (m *MockFullNode) StateActorManifestCID(arg0 context.Context, arg1 network.Version) (cid.Cid, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateActorManifestCID", arg0, arg1)
|
||||
ret0, _ := ret[0].(cid.Cid)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateActorManifestCID indicates an expected call of StateActorManifestCID.
|
||||
func (mr *MockFullNodeMockRecorder) StateActorManifestCID(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorManifestCID", reflect.TypeOf((*MockFullNode)(nil).StateActorManifestCID), arg0, arg1)
|
||||
}
|
||||
|
||||
// StateAllMinerFaults mocks base method.
|
||||
func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2249,10 +2066,10 @@ func (mr *MockFullNodeMockRecorder) StateCall(arg0, arg1, arg2 interface{}) *gom
|
||||
}
|
||||
|
||||
// StateChangedActors mocks base method.
|
||||
func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.ActorV5, error) {
|
||||
func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.Actor, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateChangedActors", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(map[string]types.ActorV5)
|
||||
ret0, _ := ret[0].(map[string]types.Actor)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -2324,10 +2141,10 @@ func (mr *MockFullNodeMockRecorder) StateDecodeParams(arg0, arg1, arg2, arg3, ar
|
||||
}
|
||||
|
||||
// StateGetActor mocks base method.
|
||||
func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.ActorV5, error) {
|
||||
func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.Actor, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetActor", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(*types.ActorV5)
|
||||
ret0, _ := ret[0].(*types.Actor)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -2338,126 +2155,6 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StateGetAllocation mocks base method.
|
||||
func (m *MockFullNode) StateGetAllocation(arg0 context.Context, arg1 address.Address, arg2 verifreg.AllocationId, arg3 types.TipSetKey) (*verifreg.Allocation, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetAllocation", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(*verifreg.Allocation)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetAllocation indicates an expected call of StateGetAllocation.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetAllocation(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocation", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocation), arg0, arg1, arg2, arg3)
|
||||
}
|
||||
|
||||
// StateGetAllocationForPendingDeal mocks base method.
|
||||
func (m *MockFullNode) StateGetAllocationForPendingDeal(arg0 context.Context, arg1 abi.DealID, arg2 types.TipSetKey) (*verifreg.Allocation, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetAllocationForPendingDeal", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(*verifreg.Allocation)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetAllocationForPendingDeal indicates an expected call of StateGetAllocationForPendingDeal.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetAllocationForPendingDeal(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocationForPendingDeal", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocationForPendingDeal), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StateGetAllocations mocks base method.
|
||||
func (m *MockFullNode) StateGetAllocations(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.AllocationId]verifreg.Allocation, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetAllocations", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(map[verifreg.AllocationId]verifreg.Allocation)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetAllocations indicates an expected call of StateGetAllocations.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetAllocations(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocations", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocations), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StateGetClaim mocks base method.
|
||||
func (m *MockFullNode) StateGetClaim(arg0 context.Context, arg1 address.Address, arg2 verifreg.ClaimId, arg3 types.TipSetKey) (*verifreg.Claim, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetClaim", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(*verifreg.Claim)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetClaim indicates an expected call of StateGetClaim.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetClaim(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaim", reflect.TypeOf((*MockFullNode)(nil).StateGetClaim), arg0, arg1, arg2, arg3)
|
||||
}
|
||||
|
||||
// StateGetClaims mocks base method.
|
||||
func (m *MockFullNode) StateGetClaims(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.ClaimId]verifreg.Claim, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetClaims", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(map[verifreg.ClaimId]verifreg.Claim)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetClaims indicates an expected call of StateGetClaims.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetClaims(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaims", reflect.TypeOf((*MockFullNode)(nil).StateGetClaims), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
// StateGetNetworkParams mocks base method.
|
||||
func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetNetworkParams", arg0)
|
||||
ret0, _ := ret[0].(*api.NetworkParams)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetNetworkParams indicates an expected call of StateGetNetworkParams.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetNetworkParams(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetNetworkParams", reflect.TypeOf((*MockFullNode)(nil).StateGetNetworkParams), arg0)
|
||||
}
|
||||
|
||||
// StateGetRandomnessFromBeacon mocks base method.
|
||||
func (m *MockFullNode) StateGetRandomnessFromBeacon(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetRandomnessFromBeacon", arg0, arg1, arg2, arg3, arg4)
|
||||
ret0, _ := ret[0].(abi.Randomness)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetRandomnessFromBeacon indicates an expected call of StateGetRandomnessFromBeacon.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromBeacon(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromBeacon", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromBeacon), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// StateGetRandomnessFromTickets mocks base method.
|
||||
func (m *MockFullNode) StateGetRandomnessFromTickets(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateGetRandomnessFromTickets", arg0, arg1, arg2, arg3, arg4)
|
||||
ret0, _ := ret[0].(abi.Randomness)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StateGetRandomnessFromTickets indicates an expected call of StateGetRandomnessFromTickets.
|
||||
func (mr *MockFullNodeMockRecorder) StateGetRandomnessFromTickets(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetRandomnessFromTickets", reflect.TypeOf((*MockFullNode)(nil).StateGetRandomnessFromTickets), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// StateGetReceipt mocks base method.
|
||||
func (m *MockFullNode) StateGetReceipt(arg0 context.Context, arg1 cid.Cid, arg2 types.TipSetKey) (*types.MessageReceipt, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2549,10 +2246,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
|
||||
}
|
||||
|
||||
// StateMarketDeals mocks base method.
|
||||
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
|
||||
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
|
||||
ret0, _ := ret[0].(map[string]*api.MarketDeal)
|
||||
ret0, _ := ret[0].(map[string]api.MarketDeal)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -2654,10 +2351,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
|
||||
}
|
||||
|
||||
// StateMinerInfo mocks base method.
|
||||
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
|
||||
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
|
||||
ret0, _ := ret[0].(api.MinerInfo)
|
||||
ret0, _ := ret[0].(miner.MinerInfo)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -2669,7 +2366,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
|
||||
}
|
||||
|
||||
// StateMinerInitialPledgeCollateral mocks base method.
|
||||
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
|
||||
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(big.Int)
|
||||
@@ -2714,7 +2411,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
|
||||
}
|
||||
|
||||
// StateMinerPreCommitDepositForPower mocks base method.
|
||||
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
|
||||
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(big.Int)
|
||||
@@ -2894,10 +2591,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsgLimited(arg0, arg1, arg2 inter
|
||||
}
|
||||
|
||||
// StateSectorExpiration mocks base method.
|
||||
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
|
||||
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(*miner0.SectorExpiration)
|
||||
ret0, _ := ret[0].(*miner.SectorExpiration)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
@@ -2924,10 +2621,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
|
||||
}
|
||||
|
||||
// StateSectorPartition mocks base method.
|
||||
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
|
||||
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
|
||||
ret0, _ := ret[0].(*miner0.SectorLocation)
|
||||
ret0, _ := ret[0].(*miner.SectorLocation)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
+5
-187
@@ -3,40 +3,22 @@ package v0api
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/v1api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
marketevents "github.com/filecoin-project/lotus/markets/loggers"
|
||||
)
|
||||
|
||||
type WrapperV1Full struct {
|
||||
v1api.FullNode
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
|
||||
pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
|
||||
if err != nil {
|
||||
return miner.SectorPreCommitOnChainInfo{}, err
|
||||
}
|
||||
if pi == nil {
|
||||
return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
|
||||
}
|
||||
|
||||
return *pi, nil
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
|
||||
return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
|
||||
}
|
||||
@@ -124,7 +106,7 @@ func (w *WrapperV1Full) MsigApproveTxnHash(ctx context.Context, msig address.Add
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) MsigCancel(ctx context.Context, msig address.Address, txID uint64, to address.Address, amt types.BigInt, src address.Address, method uint64, params []byte) (cid.Cid, error) {
|
||||
p, err := w.FullNode.MsigCancelTxnHash(ctx, msig, txID, to, amt, src, method, params)
|
||||
p, err := w.FullNode.MsigCancel(ctx, msig, txID, to, amt, src, method, params)
|
||||
if err != nil {
|
||||
return cid.Undef, xerrors.Errorf("creating prototype: %w", err)
|
||||
}
|
||||
@@ -202,168 +184,4 @@ func (w *WrapperV1Full) MsigRemoveSigner(ctx context.Context, msig address.Addre
|
||||
return w.executePrototype(ctx, p)
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
|
||||
return w.StateGetRandomnessFromTickets(ctx, personalization, randEpoch, entropy, tsk)
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) {
|
||||
return w.StateGetRandomnessFromBeacon(ctx, personalization, randEpoch, entropy, tsk)
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef) error {
|
||||
events := make(chan marketevents.RetrievalEvent)
|
||||
go w.clientRetrieve(ctx, order, ref, events)
|
||||
|
||||
for {
|
||||
select {
|
||||
case evt, ok := <-events:
|
||||
if !ok { // done successfully
|
||||
return nil
|
||||
}
|
||||
|
||||
if evt.Err != "" {
|
||||
return xerrors.Errorf("retrieval failed: %s", evt.Err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return xerrors.Errorf("retrieval timed out")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
|
||||
events := make(chan marketevents.RetrievalEvent)
|
||||
go w.clientRetrieve(ctx, order, ref, events)
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents <-chan api.RetrievalInfo, events chan marketevents.RetrievalEvent) error {
|
||||
for {
|
||||
var subscribeEvent api.RetrievalInfo
|
||||
var evt retrievalmarket.ClientEvent
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return xerrors.New("Retrieval Timed Out")
|
||||
case subscribeEvent = <-subscribeEvents:
|
||||
if subscribeEvent.ID != dealID {
|
||||
// we can't check the deal ID ahead of time because:
|
||||
// 1. We need to subscribe before retrieving.
|
||||
// 2. We won't know the deal ID until after retrieving.
|
||||
continue
|
||||
}
|
||||
if subscribeEvent.Event != nil {
|
||||
evt = *subscribeEvent.Event
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return xerrors.New("Retrieval Timed Out")
|
||||
case events <- marketevents.RetrievalEvent{
|
||||
Event: evt,
|
||||
Status: subscribeEvent.Status,
|
||||
BytesReceived: subscribeEvent.BytesReceived,
|
||||
FundsSpent: subscribeEvent.TotalPaid,
|
||||
}:
|
||||
}
|
||||
|
||||
switch subscribeEvent.Status {
|
||||
case retrievalmarket.DealStatusCompleted:
|
||||
return nil
|
||||
case retrievalmarket.DealStatusRejected:
|
||||
return xerrors.Errorf("Retrieval Proposal Rejected: %s", subscribeEvent.Message)
|
||||
case
|
||||
retrievalmarket.DealStatusDealNotFound,
|
||||
retrievalmarket.DealStatusErrored:
|
||||
return xerrors.Errorf("Retrieval Error: %s", subscribeEvent.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) clientRetrieve(ctx context.Context, order RetrievalOrder, ref *api.FileRef, events chan marketevents.RetrievalEvent) {
|
||||
defer close(events)
|
||||
|
||||
finish := func(e error) {
|
||||
if e != nil {
|
||||
events <- marketevents.RetrievalEvent{Err: e.Error(), FundsSpent: big.Zero()}
|
||||
}
|
||||
}
|
||||
|
||||
var dealID retrievalmarket.DealID
|
||||
if order.FromLocalCAR == "" {
|
||||
// Subscribe to events before retrieving to avoid losing events.
|
||||
subscribeCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
retrievalEvents, err := w.ClientGetRetrievalUpdates(subscribeCtx)
|
||||
|
||||
if err != nil {
|
||||
finish(xerrors.Errorf("GetRetrievalUpdates failed: %w", err))
|
||||
return
|
||||
}
|
||||
|
||||
retrievalRes, err := w.FullNode.ClientRetrieve(ctx, api.RetrievalOrder{
|
||||
Root: order.Root,
|
||||
Piece: order.Piece,
|
||||
Size: order.Size,
|
||||
Total: order.Total,
|
||||
UnsealPrice: order.UnsealPrice,
|
||||
PaymentInterval: order.PaymentInterval,
|
||||
PaymentIntervalIncrease: order.PaymentIntervalIncrease,
|
||||
Client: order.Client,
|
||||
Miner: order.Miner,
|
||||
MinerPeer: order.MinerPeer,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
finish(xerrors.Errorf("Retrieve failed: %w", err))
|
||||
return
|
||||
}
|
||||
|
||||
dealID = retrievalRes.DealID
|
||||
|
||||
err = readSubscribeEvents(ctx, retrievalRes.DealID, retrievalEvents, events)
|
||||
if err != nil {
|
||||
finish(xerrors.Errorf("Retrieve: %w", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// If ref is nil, it only fetches the data into the configured blockstore.
|
||||
if ref == nil {
|
||||
finish(nil)
|
||||
return
|
||||
}
|
||||
|
||||
eref := api.ExportRef{
|
||||
Root: order.Root,
|
||||
FromLocalCAR: order.FromLocalCAR,
|
||||
DealID: dealID,
|
||||
}
|
||||
|
||||
if order.DatamodelPathSelector != nil {
|
||||
s := api.Selector(*order.DatamodelPathSelector)
|
||||
eref.DAGs = append(eref.DAGs, api.DagSpec{
|
||||
DataSelector: &s,
|
||||
ExportMerkleProof: true,
|
||||
})
|
||||
}
|
||||
|
||||
finish(w.ClientExport(ctx, eref, *ref))
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
|
||||
return w.FullNode.PaychFund(ctx, from, to, amt)
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
|
||||
a, err := w.FullNode.ClientQueryAsk(ctx, p, miner)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Response, nil
|
||||
}
|
||||
|
||||
func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
|
||||
return w.StateGetBeaconEntry(ctx, epoch)
|
||||
}
|
||||
|
||||
var _ FullNode = &WrapperV1Full{}
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
type FullNode = api.FullNode
|
||||
type FullNodeStruct = api.FullNodeStruct
|
||||
|
||||
type RawFullNodeAPI FullNode
|
||||
|
||||
func PermissionedFullAPI(a FullNode) FullNode {
|
||||
return api.PermissionedFullAPI(a)
|
||||
}
|
||||
|
||||
+5
-5
@@ -3,7 +3,7 @@ package api
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
xerrors "golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type Version uint32
|
||||
@@ -54,11 +54,11 @@ func VersionForType(nodeType NodeType) (Version, error) {
|
||||
|
||||
// semver versions of the rpc api exposed
|
||||
var (
|
||||
FullAPIVersion0 = newVer(1, 5, 0)
|
||||
FullAPIVersion1 = newVer(2, 3, 0)
|
||||
FullAPIVersion0 = newVer(1, 3, 1)
|
||||
FullAPIVersion1 = newVer(2, 1, 1)
|
||||
|
||||
MinerAPIVersion0 = newVer(1, 5, 0)
|
||||
WorkerAPIVersion0 = newVer(1, 7, 0)
|
||||
MinerAPIVersion0 = newVer(1, 2, 0)
|
||||
WorkerAPIVersion0 = newVer(1, 1, 0)
|
||||
)
|
||||
|
||||
//nolint:varcheck,deadcode
|
||||
|
||||
+11
-18
@@ -11,7 +11,6 @@ import (
|
||||
type ChainIO interface {
|
||||
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
|
||||
ChainHasObj(context.Context, cid.Cid) (bool, error)
|
||||
ChainPutObj(context.Context, blocks.Block) error
|
||||
}
|
||||
|
||||
type apiBlockstore struct {
|
||||
@@ -26,42 +25,36 @@ func NewAPIBlockstore(cio ChainIO) Blockstore {
|
||||
return Adapt(bs) // return an adapted blockstore.
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) DeleteBlock(context.Context, cid.Cid) error {
|
||||
func (a *apiBlockstore) DeleteBlock(cid.Cid) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
return a.api.ChainHasObj(ctx, c)
|
||||
func (a *apiBlockstore) Has(c cid.Cid) (bool, error) {
|
||||
return a.api.ChainHasObj(context.TODO(), c)
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
|
||||
bb, err := a.api.ChainReadObj(ctx, c)
|
||||
func (a *apiBlockstore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blocks.NewBlockWithCid(bb, c)
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
bb, err := a.api.ChainReadObj(ctx, c)
|
||||
func (a *apiBlockstore) GetSize(c cid.Cid) (int, error) {
|
||||
bb, err := a.api.ChainReadObj(context.TODO(), c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(bb), nil
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) Put(ctx context.Context, block blocks.Block) error {
|
||||
return a.api.ChainPutObj(ctx, block)
|
||||
func (a *apiBlockstore) Put(blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
for _, block := range blocks {
|
||||
err := a.api.ChainPutObj(ctx, block)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
func (a *apiBlockstore) PutMany([]blocks.Block) error {
|
||||
return xerrors.New("not supported")
|
||||
}
|
||||
|
||||
func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
// autolog is a logger for the autobatching blockstore. It is subscoped from the
|
||||
// blockstore logger.
|
||||
var autolog = log.Named("auto")
|
||||
|
||||
// contains the same set of blocks twice, once as an ordered list for flushing, and as a map for fast access
|
||||
type blockBatch struct {
|
||||
blockList []block.Block
|
||||
blockMap map[cid.Cid]block.Block
|
||||
}
|
||||
|
||||
type AutobatchBlockstore struct {
|
||||
// TODO: drop if memory consumption is too high
|
||||
addedCids map[cid.Cid]struct{}
|
||||
|
||||
stateLock sync.Mutex
|
||||
bufferedBatch blockBatch
|
||||
|
||||
flushingBatch blockBatch
|
||||
flushErr error
|
||||
|
||||
flushCh chan struct{}
|
||||
|
||||
doFlushLock sync.Mutex
|
||||
flushRetryDelay time.Duration
|
||||
doneCh chan struct{}
|
||||
shutdown context.CancelFunc
|
||||
|
||||
backingBs Blockstore
|
||||
|
||||
bufferCapacity int
|
||||
bufferSize int
|
||||
}
|
||||
|
||||
func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
bs := &AutobatchBlockstore{
|
||||
addedCids: make(map[cid.Cid]struct{}),
|
||||
backingBs: backingBs,
|
||||
bufferCapacity: bufferCapacity,
|
||||
flushCh: make(chan struct{}, 1),
|
||||
doneCh: make(chan struct{}),
|
||||
// could be made configable
|
||||
flushRetryDelay: time.Millisecond * 100,
|
||||
shutdown: cancel,
|
||||
}
|
||||
|
||||
bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block)
|
||||
|
||||
go bs.flushWorker(ctx)
|
||||
|
||||
return bs
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error {
|
||||
bs.stateLock.Lock()
|
||||
defer bs.stateLock.Unlock()
|
||||
|
||||
_, ok := bs.addedCids[blk.Cid()]
|
||||
if !ok {
|
||||
bs.addedCids[blk.Cid()] = struct{}{}
|
||||
bs.bufferedBatch.blockList = append(bs.bufferedBatch.blockList, blk)
|
||||
bs.bufferedBatch.blockMap[blk.Cid()] = blk
|
||||
bs.bufferSize += len(blk.RawData())
|
||||
if bs.bufferSize >= bs.bufferCapacity {
|
||||
// signal that a flush is appropriate, may be ignored
|
||||
select {
|
||||
case bs.flushCh <- struct{}{}:
|
||||
default:
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) flushWorker(ctx context.Context) {
|
||||
defer close(bs.doneCh)
|
||||
for {
|
||||
select {
|
||||
case <-bs.flushCh:
|
||||
// TODO: check if we _should_ actually flush. We could get a spurious wakeup
|
||||
// here.
|
||||
putErr := bs.doFlush(ctx, false)
|
||||
for putErr != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(bs.flushRetryDelay):
|
||||
autolog.Errorf("FLUSH ERRORED: %w, retrying after %v", putErr, bs.flushRetryDelay)
|
||||
putErr = bs.doFlush(ctx, true)
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
// Do one last flush.
|
||||
_ = bs.doFlush(ctx, false)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// caller must NOT hold stateLock
|
||||
// set retryOnly to true to only retry a failed flush and not flush anything new.
|
||||
func (bs *AutobatchBlockstore) doFlush(ctx context.Context, retryOnly bool) error {
|
||||
bs.doFlushLock.Lock()
|
||||
defer bs.doFlushLock.Unlock()
|
||||
|
||||
// If we failed to flush last time, try flushing again.
|
||||
if bs.flushErr != nil {
|
||||
bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
|
||||
}
|
||||
|
||||
// If we failed, or we're _only_ retrying, bail.
|
||||
if retryOnly || bs.flushErr != nil {
|
||||
return bs.flushErr
|
||||
}
|
||||
|
||||
// Then take the current batch...
|
||||
bs.stateLock.Lock()
|
||||
// We do NOT clear addedCids here, because its purpose is to expedite Puts
|
||||
bs.flushingBatch = bs.bufferedBatch
|
||||
bs.bufferedBatch.blockList = make([]block.Block, 0, len(bs.flushingBatch.blockList))
|
||||
bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block, len(bs.flushingBatch.blockMap))
|
||||
bs.stateLock.Unlock()
|
||||
|
||||
// And try to flush it.
|
||||
bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList)
|
||||
|
||||
// If we succeeded, reset the batch. Otherwise, we'll try again next time.
|
||||
if bs.flushErr == nil {
|
||||
bs.stateLock.Lock()
|
||||
bs.flushingBatch = blockBatch{}
|
||||
bs.stateLock.Unlock()
|
||||
}
|
||||
|
||||
return bs.flushErr
|
||||
}
|
||||
|
||||
// caller must NOT hold stateLock
|
||||
func (bs *AutobatchBlockstore) Flush(ctx context.Context) error {
|
||||
return bs.doFlush(ctx, false)
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error {
|
||||
// TODO: Prevent puts after we call this to avoid losing data.
|
||||
bs.shutdown()
|
||||
select {
|
||||
case <-bs.doneCh:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
bs.doFlushLock.Lock()
|
||||
defer bs.doFlushLock.Unlock()
|
||||
|
||||
return bs.flushErr
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
|
||||
// may seem backward to check the backingBs first, but that is the likeliest case
|
||||
blk, err := bs.backingBs.Get(ctx, c)
|
||||
if err == nil {
|
||||
return blk, nil
|
||||
}
|
||||
|
||||
if !ipld.IsNotFound(err) {
|
||||
return blk, err
|
||||
}
|
||||
|
||||
bs.stateLock.Lock()
|
||||
v, ok := bs.flushingBatch.blockMap[c]
|
||||
if ok {
|
||||
bs.stateLock.Unlock()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
v, ok = bs.bufferedBatch.blockMap[c]
|
||||
if ok {
|
||||
bs.stateLock.Unlock()
|
||||
return v, nil
|
||||
}
|
||||
bs.stateLock.Unlock()
|
||||
|
||||
// We have to check the backing store one more time because it may have been flushed by the
|
||||
// time we were able to take the lock above.
|
||||
return bs.backingBs.Get(ctx, c)
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
|
||||
// if we wanted to support this, we would have to:
|
||||
// - flush
|
||||
// - delete from the backingBs (if present)
|
||||
// - remove from addedCids (if present)
|
||||
// - if present in addedCids, also walk the ordered lists and remove if present
|
||||
return xerrors.New("deletion is unsupported")
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
// see note in DeleteBlock()
|
||||
return xerrors.New("deletion is unsupported")
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
_, err := bs.Get(ctx, c)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if ipld.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
blk, err := bs.Get(ctx, c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return len(blk.RawData()), nil
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
|
||||
for _, blk := range blks {
|
||||
if err := bs.Put(ctx, blk); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
if err := bs.Flush(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bs.backingBs.AllKeysChan(ctx)
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) HashOnRead(enabled bool) {
|
||||
bs.backingBs.HashOnRead(enabled)
|
||||
}
|
||||
|
||||
func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
|
||||
blk, err := bs.Get(ctx, cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return callback(blk.RawData())
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAutobatchBlockstore(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ab := NewAutobatch(ctx, NewMemory(), len(b0.RawData())+len(b1.RawData())-1)
|
||||
|
||||
require.NoError(t, ab.Put(ctx, b0))
|
||||
require.NoError(t, ab.Put(ctx, b1))
|
||||
require.NoError(t, ab.Put(ctx, b2))
|
||||
|
||||
v0, err := ab.Get(ctx, b0.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, b0.RawData(), v0.RawData())
|
||||
|
||||
v1, err := ab.Get(ctx, b1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, b1.RawData(), v1.RawData())
|
||||
|
||||
v2, err := ab.Get(ctx, b2.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, b2.RawData(), v2.RawData())
|
||||
|
||||
// Regression test for a deadlock.
|
||||
_, err = ab.Get(ctx, b3.Cid())
|
||||
require.True(t, ipld.IsNotFound(err))
|
||||
|
||||
require.NoError(t, ab.Flush(ctx))
|
||||
require.NoError(t, ab.Shutdown(ctx))
|
||||
}
|
||||
+23
-137
@@ -13,14 +13,13 @@ import (
|
||||
"github.com/dgraph-io/badger/v2"
|
||||
"github.com/dgraph-io/badger/v2/options"
|
||||
"github.com/dgraph-io/badger/v2/pb"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logger "github.com/ipfs/go-log/v2"
|
||||
pool "github.com/libp2p/go-buffer-pool"
|
||||
"github.com/multiformats/go-base32"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
logger "github.com/ipfs/go-log/v2"
|
||||
pool "github.com/libp2p/go-buffer-pool"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
@@ -45,8 +44,7 @@ const (
|
||||
// MemoryMap is equivalent to badger/options.MemoryMap.
|
||||
MemoryMap = options.MemoryMap
|
||||
// LoadToRAM is equivalent to badger/options.LoadToRAM.
|
||||
LoadToRAM = options.LoadToRAM
|
||||
defaultGCThreshold = 0.125
|
||||
LoadToRAM = options.LoadToRAM
|
||||
)
|
||||
|
||||
// Options embeds the badger options themselves, and augments them with
|
||||
@@ -396,9 +394,6 @@ func (b *Blockstore) doCopy(from, to *badger.DB) error {
|
||||
if workers < 2 {
|
||||
workers = 2
|
||||
}
|
||||
if workers > 8 {
|
||||
workers = 8
|
||||
}
|
||||
|
||||
stream := from.NewStream()
|
||||
stream.NumGo = workers
|
||||
@@ -444,7 +439,7 @@ func (b *Blockstore) deleteDB(path string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq time.Duration, check func() error) error {
|
||||
func (b *Blockstore) onlineGC() error {
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
@@ -453,26 +448,14 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
|
||||
if nworkers < 2 {
|
||||
nworkers = 2
|
||||
}
|
||||
if nworkers > 7 { // max out at 1 goroutine per badger level
|
||||
nworkers = 7
|
||||
}
|
||||
|
||||
err := b.db.Flatten(nworkers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checkTick := time.NewTimer(checkFreq)
|
||||
defer checkTick.Stop()
|
||||
|
||||
for err == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-checkTick.C:
|
||||
err = check()
|
||||
checkTick.Reset(checkFreq)
|
||||
default:
|
||||
err = b.db.RunValueLogGC(threshold)
|
||||
}
|
||||
err = b.db.RunValueLogGC(0.125)
|
||||
}
|
||||
|
||||
if err == badger.ErrNoRewrite {
|
||||
@@ -485,7 +468,7 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
|
||||
|
||||
// CollectGarbage compacts and runs garbage collection on the value log;
|
||||
// implements the BlockstoreGC trait
|
||||
func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
|
||||
func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -502,58 +485,8 @@ func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.Bloc
|
||||
if options.FullGC {
|
||||
return b.movingGC()
|
||||
}
|
||||
threshold := options.Threshold
|
||||
if threshold == 0 {
|
||||
threshold = defaultGCThreshold
|
||||
}
|
||||
checkFreq := options.CheckFreq
|
||||
if checkFreq < 30*time.Second { // disallow checking more frequently than block time
|
||||
checkFreq = 30 * time.Second
|
||||
}
|
||||
check := options.Check
|
||||
if check == nil {
|
||||
check = func() error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return b.onlineGC(ctx, threshold, checkFreq, check)
|
||||
}
|
||||
|
||||
// GCOnce runs garbage collection on the value log;
|
||||
// implements BlockstoreGCOnce trait
|
||||
func (b *Blockstore) GCOnce(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
var options blockstore.BlockstoreGCOptions
|
||||
for _, opt := range opts {
|
||||
err := opt(&options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if options.FullGC {
|
||||
return xerrors.Errorf("FullGC option specified for GCOnce but full GC is non incremental")
|
||||
}
|
||||
|
||||
threshold := options.Threshold
|
||||
if threshold == 0 {
|
||||
threshold = defaultGCThreshold
|
||||
}
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
// Note no compaction needed before single GC as we will hit at most one vlog anyway
|
||||
err := b.db.RunValueLogGC(threshold)
|
||||
if err == badger.ErrNoRewrite {
|
||||
// not really an error in this case, it signals the end of GC
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
return b.onlineGC()
|
||||
}
|
||||
|
||||
// Size returns the aggregate size of the blockstore
|
||||
@@ -592,7 +525,7 @@ func (b *Blockstore) Size() (int64, error) {
|
||||
|
||||
// View implements blockstore.Viewer, which leverages zero-copy read-only
|
||||
// access to values.
|
||||
func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) error) error {
|
||||
func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -611,27 +544,15 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
|
||||
case nil:
|
||||
return item.Value(fn)
|
||||
case badger.ErrKeyNotFound:
|
||||
return ipld.ErrNotFound{Cid: cid}
|
||||
return blockstore.ErrNotFound
|
||||
default:
|
||||
return fmt.Errorf("failed to view block from badger blockstore: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Blockstore) Flush(context.Context) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.viewers.Done()
|
||||
|
||||
b.lockDB()
|
||||
defer b.unlockDB()
|
||||
|
||||
return b.db.Sync()
|
||||
}
|
||||
|
||||
// Has implements Blockstore.Has.
|
||||
func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
func (b *Blockstore) Has(cid cid.Cid) (bool, error) {
|
||||
if err := b.access(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -661,9 +582,9 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
}
|
||||
|
||||
// Get implements Blockstore.Get.
|
||||
func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
func (b *Blockstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
if !cid.Defined() {
|
||||
return nil, ipld.ErrNotFound{Cid: cid}
|
||||
return nil, blockstore.ErrNotFound
|
||||
}
|
||||
|
||||
if err := b.access(); err != nil {
|
||||
@@ -686,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
|
||||
val, err = item.ValueCopy(nil)
|
||||
return err
|
||||
case badger.ErrKeyNotFound:
|
||||
return ipld.ErrNotFound{Cid: cid}
|
||||
return blockstore.ErrNotFound
|
||||
default:
|
||||
return fmt.Errorf("failed to get block from badger blockstore: %w", err)
|
||||
}
|
||||
@@ -698,7 +619,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
|
||||
}
|
||||
|
||||
// GetSize implements Blockstore.GetSize.
|
||||
func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
func (b *Blockstore) GetSize(cid cid.Cid) (int, error) {
|
||||
if err := b.access(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -718,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
case nil:
|
||||
size = int(item.ValueSize())
|
||||
case badger.ErrKeyNotFound:
|
||||
return ipld.ErrNotFound{Cid: cid}
|
||||
return blockstore.ErrNotFound
|
||||
default:
|
||||
return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
|
||||
}
|
||||
@@ -731,7 +652,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
}
|
||||
|
||||
// Put implements Blockstore.Put.
|
||||
func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
|
||||
func (b *Blockstore) Put(block blocks.Block) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -746,20 +667,6 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
|
||||
}
|
||||
|
||||
put := func(db *badger.DB) error {
|
||||
// Check if we have it before writing it.
|
||||
switch err := db.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(k)
|
||||
return err
|
||||
}); err {
|
||||
case badger.ErrKeyNotFound:
|
||||
case nil:
|
||||
// Already exists, skip the put.
|
||||
return nil
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
// Then write it.
|
||||
err := db.Update(func(txn *badger.Txn) error {
|
||||
return txn.Set(k, block.RawData())
|
||||
})
|
||||
@@ -784,7 +691,7 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
|
||||
}
|
||||
|
||||
// PutMany implements Blockstore.PutMany.
|
||||
func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
func (b *Blockstore) PutMany(blocks []blocks.Block) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -815,33 +722,12 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
err := b.db.View(func(txn *badger.Txn) error {
|
||||
for i, k := range keys {
|
||||
switch _, err := txn.Get(k); err {
|
||||
case badger.ErrKeyNotFound:
|
||||
case nil:
|
||||
keys[i] = nil
|
||||
default:
|
||||
// Something is actually wrong
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
put := func(db *badger.DB) error {
|
||||
batch := db.NewWriteBatch()
|
||||
defer batch.Cancel()
|
||||
|
||||
for i, block := range blocks {
|
||||
k := keys[i]
|
||||
if k == nil {
|
||||
// skipped because we already have it.
|
||||
continue
|
||||
}
|
||||
if err := batch.Set(k, block.RawData()); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -869,7 +755,7 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
}
|
||||
|
||||
// DeleteBlock implements Blockstore.DeleteBlock.
|
||||
func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
func (b *Blockstore) DeleteBlock(cid cid.Cid) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -888,7 +774,7 @@ func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Blockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
|
||||
if err := b.access(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,26 +1,24 @@
|
||||
// stm: #unit
|
||||
package badgerbs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
|
||||
func TestBadgerBlockstore(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
(&Suite{
|
||||
NewBlockstore: newBlockstore(DefaultOptions),
|
||||
OpenBlockstore: openBlockstore(DefaultOptions),
|
||||
@@ -39,8 +37,6 @@ func TestBadgerBlockstore(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStorageKey(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_STORAGE_KEY_001
|
||||
bs, _ := newBlockstore(DefaultOptions)(t)
|
||||
bbs := bs.(*Blockstore)
|
||||
defer bbs.Close() //nolint:errcheck
|
||||
@@ -76,13 +72,20 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (
|
||||
return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) {
|
||||
tb.Helper()
|
||||
|
||||
path = tb.TempDir()
|
||||
path, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := Open(optsSupplier(path))
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
|
||||
tb.Cleanup(func() {
|
||||
_ = os.RemoveAll(path)
|
||||
})
|
||||
|
||||
return db, path
|
||||
}
|
||||
}
|
||||
@@ -95,11 +98,17 @@ func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB,
|
||||
}
|
||||
|
||||
func testMove(t *testing.T, optsF func(string) Options) {
|
||||
ctx := context.Background()
|
||||
basePath := t.TempDir()
|
||||
basePath, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(basePath, "db")
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = os.RemoveAll(basePath)
|
||||
})
|
||||
|
||||
db, err := Open(optsF(dbPath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -113,7 +122,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
// add some blocks
|
||||
for i := 0; i < 10; i++ {
|
||||
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
||||
err := db.Put(ctx, blk)
|
||||
err := db.Put(blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -123,7 +132,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
// delete some of them
|
||||
for i := 5; i < 10; i++ {
|
||||
c := have[i].Cid()
|
||||
err := db.DeleteBlock(ctx, c)
|
||||
err := db.DeleteBlock(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -136,7 +145,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
g.Go(func() error {
|
||||
for i := 10; i < 1000; i++ {
|
||||
blk := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
||||
err := db.Put(ctx, blk)
|
||||
err := db.Put(blk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -145,7 +154,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
return nil
|
||||
})
|
||||
g.Go(func() error {
|
||||
return db.CollectGarbage(ctx, blockstore.WithFullGC(true))
|
||||
return db.CollectGarbage(blockstore.WithFullGC(true))
|
||||
})
|
||||
|
||||
err = g.Wait()
|
||||
@@ -156,7 +165,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
// now check that we have all the blocks in have and none in the deleted lists
|
||||
checkBlocks := func() {
|
||||
for _, blk := range have {
|
||||
has, err := db.Has(ctx, blk.Cid())
|
||||
has, err := db.Has(blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -165,7 +174,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
t.Fatal("missing block")
|
||||
}
|
||||
|
||||
blk2, err := db.Get(ctx, blk.Cid())
|
||||
blk2, err := db.Get(blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -176,7 +185,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
}
|
||||
|
||||
for _, c := range deleted {
|
||||
has, err := db.Has(ctx, c)
|
||||
has, err := db.Has(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -230,7 +239,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
checkPath()
|
||||
|
||||
// now do another FullGC to test the double move and following of symlinks
|
||||
if err := db.CollectGarbage(ctx, blockstore.WithFullGC(true)); err != nil {
|
||||
if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -254,16 +263,10 @@ func testMove(t *testing.T, optsF func(string) Options) {
|
||||
}
|
||||
|
||||
func TestMoveNoPrefix(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
|
||||
testMove(t, DefaultOptions)
|
||||
}
|
||||
|
||||
func TestMoveWithPrefix(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001
|
||||
testMove(t, func(path string) Options {
|
||||
opts := DefaultOptions(path)
|
||||
opts.Prefix = "/prefixed/"
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
// stm: #unit
|
||||
package badgerbs
|
||||
|
||||
import (
|
||||
@@ -9,13 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
u "github.com/ipfs/boxo/util"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/stretchr/testify/require"
|
||||
u "github.com/ipfs/go-ipfs-util"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TODO: move this to go-ipfs-blockstore.
|
||||
@@ -45,38 +44,28 @@ func (s *Suite) RunTests(t *testing.T, prefix string) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
}
|
||||
|
||||
c := cid.NewCidV0(u.Hash([]byte("stuff")))
|
||||
bl, err := bs.Get(ctx, c)
|
||||
bl, err := bs.Get(c)
|
||||
require.Nil(t, bl)
|
||||
require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
|
||||
require.Equal(t, blockstore.ErrNotFound, err)
|
||||
}
|
||||
|
||||
func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
}
|
||||
|
||||
_, err := bs.Get(ctx, cid.Undef)
|
||||
require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
|
||||
_, err := bs.Get(cid.Undef)
|
||||
require.Equal(t, blockstore.ErrNotFound, err)
|
||||
}
|
||||
|
||||
func (s *Suite) TestPutThenGetBlock(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -84,18 +73,15 @@ func (s *Suite) TestPutThenGetBlock(t *testing.T) {
|
||||
|
||||
orig := blocks.NewBlock([]byte("some data"))
|
||||
|
||||
err := bs.Put(ctx, orig)
|
||||
err := bs.Put(orig)
|
||||
require.NoError(t, err)
|
||||
|
||||
fetched, err := bs.Get(ctx, orig.Cid())
|
||||
fetched, err := bs.Get(orig.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, orig.RawData(), fetched.RawData())
|
||||
}
|
||||
|
||||
func (s *Suite) TestHas(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -103,23 +89,19 @@ func (s *Suite) TestHas(t *testing.T) {
|
||||
|
||||
orig := blocks.NewBlock([]byte("some data"))
|
||||
|
||||
err := bs.Put(ctx, orig)
|
||||
err := bs.Put(orig)
|
||||
require.NoError(t, err)
|
||||
|
||||
ok, err := bs.Has(ctx, orig.Cid())
|
||||
ok, err := bs.Has(orig.Cid())
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
ok, err = bs.Has(ctx, blocks.NewBlock([]byte("another thing")).Cid())
|
||||
ok, err = bs.Has(blocks.NewBlock([]byte("another thing")).Cid())
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func (s *Suite) TestCidv0v1(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -127,20 +109,15 @@ func (s *Suite) TestCidv0v1(t *testing.T) {
|
||||
|
||||
orig := blocks.NewBlock([]byte("some data"))
|
||||
|
||||
err := bs.Put(ctx, orig)
|
||||
err := bs.Put(orig)
|
||||
require.NoError(t, err)
|
||||
|
||||
fetched, err := bs.Get(ctx, cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
|
||||
fetched, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, orig.Cid().Hash()))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, orig.RawData(), fetched.RawData())
|
||||
}
|
||||
|
||||
func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_SIZE_001
|
||||
ctx := context.Background()
|
||||
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -150,29 +127,26 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
|
||||
missingBlock := blocks.NewBlock([]byte("missingBlock"))
|
||||
emptyBlock := blocks.NewBlock([]byte{})
|
||||
|
||||
err := bs.Put(ctx, block)
|
||||
err := bs.Put(block)
|
||||
require.NoError(t, err)
|
||||
|
||||
blockSize, err := bs.GetSize(ctx, block.Cid())
|
||||
blockSize, err := bs.GetSize(block.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, block.RawData(), blockSize)
|
||||
|
||||
err = bs.Put(ctx, emptyBlock)
|
||||
err = bs.Put(emptyBlock)
|
||||
require.NoError(t, err)
|
||||
|
||||
emptySize, err := bs.GetSize(ctx, emptyBlock.Cid())
|
||||
emptySize, err := bs.GetSize(emptyBlock.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, emptySize)
|
||||
|
||||
missingCid := missingBlock.Cid()
|
||||
missingSize, err := bs.GetSize(ctx, missingCid)
|
||||
require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
|
||||
missingSize, err := bs.GetSize(missingBlock.Cid())
|
||||
require.Equal(t, blockstore.ErrNotFound, err)
|
||||
require.Equal(t, -1, missingSize)
|
||||
}
|
||||
|
||||
func (s *Suite) TestAllKeysSimple(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -189,9 +163,6 @@ func (s *Suite) TestAllKeysSimple(t *testing.T) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -222,7 +193,6 @@ func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestDoubleClose(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
c, ok := bs.(io.Closer)
|
||||
if !ok {
|
||||
@@ -233,10 +203,6 @@ func (s *Suite) TestDoubleClose(t *testing.T) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestReopenPutGet(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001
|
||||
ctx := context.Background()
|
||||
bs, path := s.NewBlockstore(t)
|
||||
c, ok := bs.(io.Closer)
|
||||
if !ok {
|
||||
@@ -244,7 +210,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
|
||||
}
|
||||
|
||||
orig := blocks.NewBlock([]byte("some data"))
|
||||
err := bs.Put(ctx, orig)
|
||||
err := bs.Put(orig)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = c.Close()
|
||||
@@ -253,7 +219,7 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
|
||||
bs, err = s.OpenBlockstore(t, path)
|
||||
require.NoError(t, err)
|
||||
|
||||
fetched, err := bs.Get(ctx, orig.Cid())
|
||||
fetched, err := bs.Get(orig.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, orig.RawData(), fetched.RawData())
|
||||
|
||||
@@ -262,11 +228,6 @@ func (s *Suite) TestReopenPutGet(t *testing.T) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestPutMany(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001
|
||||
//stm: @SPLITSTORE_BADGER_HAS_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_GET_001, @SPLITSTORE_BADGER_PUT_MANY_001
|
||||
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -277,15 +238,15 @@ func (s *Suite) TestPutMany(t *testing.T) {
|
||||
blocks.NewBlock([]byte("foo2")),
|
||||
blocks.NewBlock([]byte("foo3")),
|
||||
}
|
||||
err := bs.PutMany(ctx, blks)
|
||||
err := bs.PutMany(blks)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, blk := range blks {
|
||||
fetched, err := bs.Get(ctx, blk.Cid())
|
||||
fetched, err := bs.Get(blk.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, blk.RawData(), fetched.RawData())
|
||||
|
||||
ok, err := bs.Has(ctx, blk.Cid())
|
||||
ok, err := bs.Has(blk.Cid())
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
}
|
||||
@@ -298,12 +259,6 @@ func (s *Suite) TestPutMany(t *testing.T) {
|
||||
}
|
||||
|
||||
func (s *Suite) TestDelete(t *testing.T) {
|
||||
//stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001
|
||||
//stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_POOLED_STORAGE_HAS_001
|
||||
//stm: @SPLITSTORE_BADGER_ALL_KEYS_CHAN_001, @SPLITSTORE_BADGER_HAS_001
|
||||
//stm: @SPLITSTORE_BADGER_PUT_MANY_001
|
||||
|
||||
ctx := context.Background()
|
||||
bs, _ := s.NewBlockstore(t)
|
||||
if c, ok := bs.(io.Closer); ok {
|
||||
defer func() { require.NoError(t, c.Close()) }()
|
||||
@@ -314,10 +269,10 @@ func (s *Suite) TestDelete(t *testing.T) {
|
||||
blocks.NewBlock([]byte("foo2")),
|
||||
blocks.NewBlock([]byte("foo3")),
|
||||
}
|
||||
err := bs.PutMany(ctx, blks)
|
||||
err := bs.PutMany(blks)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = bs.DeleteBlock(ctx, blks[1].Cid())
|
||||
err = bs.DeleteBlock(blks[1].Cid())
|
||||
require.NoError(t, err)
|
||||
|
||||
ch, err := bs.AllKeysChan(context.Background())
|
||||
@@ -330,17 +285,17 @@ func (s *Suite) TestDelete(t *testing.T) {
|
||||
cid.NewCidV1(cid.Raw, blks[2].Cid().Hash()),
|
||||
})
|
||||
|
||||
has, err := bs.Has(ctx, blks[1].Cid())
|
||||
has, err := bs.Has(blks[1].Cid())
|
||||
require.NoError(t, err)
|
||||
require.False(t, has)
|
||||
|
||||
}
|
||||
|
||||
func insertBlocks(t *testing.T, bs blockstore.BasicBlockstore, count int) []cid.Cid {
|
||||
ctx := context.Background()
|
||||
keys := make([]cid.Cid, count)
|
||||
for i := 0; i < count; i++ {
|
||||
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
|
||||
err := bs.Put(ctx, block)
|
||||
err := bs.Put(block)
|
||||
require.NoError(t, err)
|
||||
// NewBlock assigns a CIDv0; we convert it to CIDv1 because that's what
|
||||
// the store returns.
|
||||
|
||||
+11
-55
@@ -1,17 +1,17 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
blockstore "github.com/ipfs/boxo/blockstore"
|
||||
"github.com/ipfs/go-cid"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
ds "github.com/ipfs/go-datastore"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
|
||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||
)
|
||||
|
||||
var log = logging.Logger("blockstore")
|
||||
|
||||
var ErrNotFound = blockstore.ErrNotFound
|
||||
|
||||
// Blockstore is the blockstore interface used by Lotus. It is the union
|
||||
// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
|
||||
// e.g. View or Sync.
|
||||
@@ -19,7 +19,6 @@ type Blockstore interface {
|
||||
blockstore.Blockstore
|
||||
blockstore.Viewer
|
||||
BatchDeleter
|
||||
Flusher
|
||||
}
|
||||
|
||||
// BasicBlockstore is an alias to the original IPFS Blockstore.
|
||||
@@ -27,12 +26,8 @@ type BasicBlockstore = blockstore.Blockstore
|
||||
|
||||
type Viewer = blockstore.Viewer
|
||||
|
||||
type Flusher interface {
|
||||
Flush(context.Context) error
|
||||
}
|
||||
|
||||
type BatchDeleter interface {
|
||||
DeleteMany(ctx context.Context, cids []cid.Cid) error
|
||||
DeleteMany(cids []cid.Cid) error
|
||||
}
|
||||
|
||||
// BlockstoreIterator is a trait for efficient iteration
|
||||
@@ -42,12 +37,7 @@ type BlockstoreIterator interface {
|
||||
|
||||
// BlockstoreGC is a trait for blockstores that support online garbage collection
|
||||
type BlockstoreGC interface {
|
||||
CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error
|
||||
}
|
||||
|
||||
// BlockstoreGCOnce is a trait for a blockstore that supports incremental online garbage collection
|
||||
type BlockstoreGCOnce interface {
|
||||
GCOnce(ctx context.Context, options ...BlockstoreGCOption) error
|
||||
CollectGarbage(options ...BlockstoreGCOption) error
|
||||
}
|
||||
|
||||
// BlockstoreGCOption is a functional interface for controlling blockstore GC options
|
||||
@@ -56,12 +46,6 @@ type BlockstoreGCOption = func(*BlockstoreGCOptions) error
|
||||
// BlockstoreGCOptions is a struct with GC options
|
||||
type BlockstoreGCOptions struct {
|
||||
FullGC bool
|
||||
// fraction of garbage in badger vlog before its worth processing in online GC
|
||||
Threshold float64
|
||||
// how often to call the check function
|
||||
CheckFreq time.Duration
|
||||
// function to call periodically to pause or early terminate GC
|
||||
Check func() error
|
||||
}
|
||||
|
||||
func WithFullGC(fullgc bool) BlockstoreGCOption {
|
||||
@@ -71,27 +55,6 @@ func WithFullGC(fullgc bool) BlockstoreGCOption {
|
||||
}
|
||||
}
|
||||
|
||||
func WithThreshold(threshold float64) BlockstoreGCOption {
|
||||
return func(opts *BlockstoreGCOptions) error {
|
||||
opts.Threshold = threshold
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithCheckFreq(f time.Duration) BlockstoreGCOption {
|
||||
return func(opts *BlockstoreGCOptions) error {
|
||||
opts.CheckFreq = f
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func WithCheck(check func() error) BlockstoreGCOption {
|
||||
return func(opts *BlockstoreGCOptions) error {
|
||||
opts.Check = check
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// BlockstoreSize is a trait for on-disk blockstores that can report their size
|
||||
type BlockstoreSize interface {
|
||||
Size() (int64, error)
|
||||
@@ -130,24 +93,17 @@ type adaptedBlockstore struct {
|
||||
|
||||
var _ Blockstore = (*adaptedBlockstore)(nil)
|
||||
|
||||
func (a *adaptedBlockstore) Flush(ctx context.Context) error {
|
||||
if flusher, canFlush := a.Blockstore.(Flusher); canFlush {
|
||||
return flusher.Flush(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
|
||||
blk, err := a.Get(ctx, cid)
|
||||
func (a *adaptedBlockstore) View(cid cid.Cid, callback func([]byte) error) error {
|
||||
blk, err := a.Get(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return callback(blk.RawData())
|
||||
}
|
||||
|
||||
func (a *adaptedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
func (a *adaptedBlockstore) DeleteMany(cids []cid.Cid) error {
|
||||
for _, cid := range cids {
|
||||
err := a.DeleteBlock(ctx, cid)
|
||||
err := a.DeleteBlock(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+25
-28
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
)
|
||||
|
||||
// buflog is a logger for the buffered blockstore. It is subscoped from the
|
||||
@@ -46,8 +45,6 @@ var (
|
||||
_ Viewer = (*BufferedBlockstore)(nil)
|
||||
)
|
||||
|
||||
func (bs *BufferedBlockstore) Flush(ctx context.Context) error { return bs.write.Flush(ctx) }
|
||||
|
||||
func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
a, err := bs.read.AllKeysChan(ctx)
|
||||
if err != nil {
|
||||
@@ -91,55 +88,55 @@ func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
|
||||
if err := bs.read.DeleteBlock(ctx, c); err != nil {
|
||||
func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error {
|
||||
if err := bs.read.DeleteBlock(c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bs.write.DeleteBlock(ctx, c)
|
||||
return bs.write.DeleteBlock(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
if err := bs.read.DeleteMany(ctx, cids); err != nil {
|
||||
func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error {
|
||||
if err := bs.read.DeleteMany(cids); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bs.write.DeleteMany(ctx, cids)
|
||||
return bs.write.DeleteMany(cids)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
|
||||
func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error {
|
||||
// both stores are viewable.
|
||||
if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
|
||||
if err := bs.write.View(c, callback); err == ErrNotFound {
|
||||
// not found in write blockstore; fall through.
|
||||
} else {
|
||||
return err // propagate errors, or nil, i.e. found.
|
||||
}
|
||||
return bs.read.View(ctx, c, callback)
|
||||
return bs.read.View(c, callback)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
|
||||
if out, err := bs.write.Get(ctx, c); err != nil {
|
||||
if !ipld.IsNotFound(err) {
|
||||
func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error) {
|
||||
if out, err := bs.write.Get(c); err != nil {
|
||||
if err != ErrNotFound {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
return bs.read.Get(ctx, c)
|
||||
return bs.read.Get(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
s, err := bs.read.GetSize(ctx, c)
|
||||
if ipld.IsNotFound(err) || s == 0 {
|
||||
return bs.write.GetSize(ctx, c)
|
||||
func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error) {
|
||||
s, err := bs.read.GetSize(c)
|
||||
if err == ErrNotFound || s == 0 {
|
||||
return bs.write.GetSize(c)
|
||||
}
|
||||
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
|
||||
has, err := bs.read.Has(ctx, blk.Cid()) // TODO: consider dropping this check
|
||||
func (bs *BufferedBlockstore) Put(blk block.Block) error {
|
||||
has, err := bs.read.Has(blk.Cid()) // TODO: consider dropping this check
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -148,11 +145,11 @@ func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return bs.write.Put(ctx, blk)
|
||||
return bs.write.Put(blk)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
has, err := bs.write.Has(ctx, c)
|
||||
func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error) {
|
||||
has, err := bs.write.Has(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -160,7 +157,7 @@ func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return bs.read.Has(ctx, c)
|
||||
return bs.read.Has(c)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) HashOnRead(hor bool) {
|
||||
@@ -168,8 +165,8 @@ func (bs *BufferedBlockstore) HashOnRead(hor bool) {
|
||||
bs.write.HashOnRead(hor)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) PutMany(ctx context.Context, blks []block.Block) error {
|
||||
return bs.write.PutMany(ctx, blks)
|
||||
func (bs *BufferedBlockstore) PutMany(blks []block.Block) error {
|
||||
return bs.write.PutMany(blks)
|
||||
}
|
||||
|
||||
func (bs *BufferedBlockstore) Read() Blockstore {
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
|
||||
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
xerrors "golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
var _ = xerrors.Errorf
|
||||
var _ = cid.Undef
|
||||
var _ = math.E
|
||||
var _ = sort.Sort
|
||||
|
||||
var lengthBufNetRpcReq = []byte{132}
|
||||
|
||||
func (t *NetRpcReq) MarshalCBOR(w io.Writer) error {
|
||||
if t == nil {
|
||||
_, err := w.Write(cbg.CborNull)
|
||||
return err
|
||||
}
|
||||
|
||||
cw := cbg.NewCborWriter(w)
|
||||
|
||||
if _, err := cw.Write(lengthBufNetRpcReq); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCReqType) (uint8)
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.ID (uint64) (uint64)
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Cid ([]cid.Cid) (slice)
|
||||
if len(t.Cid) > cbg.MaxLength {
|
||||
return xerrors.Errorf("Slice value in field t.Cid was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Cid))); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range t.Cid {
|
||||
|
||||
if err := cbg.WriteCid(cw, v); err != nil {
|
||||
return xerrors.Errorf("failed to write cid field v: %w", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// t.Data ([][]uint8) (slice)
|
||||
if len(t.Data) > cbg.MaxLength {
|
||||
return xerrors.Errorf("Slice value in field t.Data was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Data))); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range t.Data {
|
||||
if len(v) > cbg.ByteArrayMaxLen {
|
||||
return xerrors.Errorf("Byte array in field v was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(v))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := cw.Write(v[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *NetRpcReq) UnmarshalCBOR(r io.Reader) (err error) {
|
||||
*t = NetRpcReq{}
|
||||
|
||||
cr := cbg.NewCborReader(r)
|
||||
|
||||
maj, extra, err := cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
}()
|
||||
|
||||
if maj != cbg.MajArray {
|
||||
return fmt.Errorf("cbor input should be of type array")
|
||||
}
|
||||
|
||||
if extra != 4 {
|
||||
return fmt.Errorf("cbor input had wrong number of fields")
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCReqType) (uint8)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if maj != cbg.MajUnsignedInt {
|
||||
return fmt.Errorf("wrong type for uint8 field")
|
||||
}
|
||||
if extra > math.MaxUint8 {
|
||||
return fmt.Errorf("integer in input was too large for uint8 field")
|
||||
}
|
||||
t.Type = NetRPCReqType(extra)
|
||||
// t.ID (uint64) (uint64)
|
||||
|
||||
{
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if maj != cbg.MajUnsignedInt {
|
||||
return fmt.Errorf("wrong type for uint64 field")
|
||||
}
|
||||
t.ID = uint64(extra)
|
||||
|
||||
}
|
||||
// t.Cid ([]cid.Cid) (slice)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if extra > cbg.MaxLength {
|
||||
return fmt.Errorf("t.Cid: array too large (%d)", extra)
|
||||
}
|
||||
|
||||
if maj != cbg.MajArray {
|
||||
return fmt.Errorf("expected cbor array")
|
||||
}
|
||||
|
||||
if extra > 0 {
|
||||
t.Cid = make([]cid.Cid, extra)
|
||||
}
|
||||
|
||||
for i := 0; i < int(extra); i++ {
|
||||
{
|
||||
var maj byte
|
||||
var extra uint64
|
||||
var err error
|
||||
_ = maj
|
||||
_ = extra
|
||||
_ = err
|
||||
|
||||
{
|
||||
|
||||
c, err := cbg.ReadCid(cr)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to read cid field t.Cid[i]: %w", err)
|
||||
}
|
||||
|
||||
t.Cid[i] = c
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// t.Data ([][]uint8) (slice)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if extra > cbg.MaxLength {
|
||||
return fmt.Errorf("t.Data: array too large (%d)", extra)
|
||||
}
|
||||
|
||||
if maj != cbg.MajArray {
|
||||
return fmt.Errorf("expected cbor array")
|
||||
}
|
||||
|
||||
if extra > 0 {
|
||||
t.Data = make([][]uint8, extra)
|
||||
}
|
||||
|
||||
for i := 0; i < int(extra); i++ {
|
||||
{
|
||||
var maj byte
|
||||
var extra uint64
|
||||
var err error
|
||||
_ = maj
|
||||
_ = extra
|
||||
_ = err
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if extra > cbg.ByteArrayMaxLen {
|
||||
return fmt.Errorf("t.Data[i]: byte array too large (%d)", extra)
|
||||
}
|
||||
if maj != cbg.MajByteString {
|
||||
return fmt.Errorf("expected byte array")
|
||||
}
|
||||
|
||||
if extra > 0 {
|
||||
t.Data[i] = make([]uint8, extra)
|
||||
}
|
||||
|
||||
if _, err := io.ReadFull(cr, t.Data[i][:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var lengthBufNetRpcResp = []byte{131}
|
||||
|
||||
func (t *NetRpcResp) MarshalCBOR(w io.Writer) error {
|
||||
if t == nil {
|
||||
_, err := w.Write(cbg.CborNull)
|
||||
return err
|
||||
}
|
||||
|
||||
cw := cbg.NewCborWriter(w)
|
||||
|
||||
if _, err := cw.Write(lengthBufNetRpcResp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCRespType) (uint8)
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.ID (uint64) (uint64)
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Data ([]uint8) (slice)
|
||||
if len(t.Data) > cbg.ByteArrayMaxLen {
|
||||
return xerrors.Errorf("Byte array in field t.Data was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Data))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := cw.Write(t.Data[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *NetRpcResp) UnmarshalCBOR(r io.Reader) (err error) {
|
||||
*t = NetRpcResp{}
|
||||
|
||||
cr := cbg.NewCborReader(r)
|
||||
|
||||
maj, extra, err := cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
}()
|
||||
|
||||
if maj != cbg.MajArray {
|
||||
return fmt.Errorf("cbor input should be of type array")
|
||||
}
|
||||
|
||||
if extra != 3 {
|
||||
return fmt.Errorf("cbor input had wrong number of fields")
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCRespType) (uint8)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if maj != cbg.MajUnsignedInt {
|
||||
return fmt.Errorf("wrong type for uint8 field")
|
||||
}
|
||||
if extra > math.MaxUint8 {
|
||||
return fmt.Errorf("integer in input was too large for uint8 field")
|
||||
}
|
||||
t.Type = NetRPCRespType(extra)
|
||||
// t.ID (uint64) (uint64)
|
||||
|
||||
{
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if maj != cbg.MajUnsignedInt {
|
||||
return fmt.Errorf("wrong type for uint64 field")
|
||||
}
|
||||
t.ID = uint64(extra)
|
||||
|
||||
}
|
||||
// t.Data ([]uint8) (slice)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if extra > cbg.ByteArrayMaxLen {
|
||||
return fmt.Errorf("t.Data: byte array too large (%d)", extra)
|
||||
}
|
||||
if maj != cbg.MajByteString {
|
||||
return fmt.Errorf("expected byte array")
|
||||
}
|
||||
|
||||
if extra > 0 {
|
||||
t.Data = make([]uint8, extra)
|
||||
}
|
||||
|
||||
if _, err := io.ReadFull(cr, t.Data[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var lengthBufNetRpcErr = []byte{131}
|
||||
|
||||
func (t *NetRpcErr) MarshalCBOR(w io.Writer) error {
|
||||
if t == nil {
|
||||
_, err := w.Write(cbg.CborNull)
|
||||
return err
|
||||
}
|
||||
|
||||
cw := cbg.NewCborWriter(w)
|
||||
|
||||
if _, err := cw.Write(lengthBufNetRpcErr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCErrType) (uint8)
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Msg (string) (string)
|
||||
if len(t.Msg) > cbg.MaxLength {
|
||||
return xerrors.Errorf("Value in field t.Msg was too long")
|
||||
}
|
||||
|
||||
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Msg))); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := cw.WriteString(string(t.Msg)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// t.Cid (cid.Cid) (struct)
|
||||
|
||||
if t.Cid == nil {
|
||||
if _, err := cw.Write(cbg.CborNull); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := cbg.WriteCid(cw, *t.Cid); err != nil {
|
||||
return xerrors.Errorf("failed to write cid field t.Cid: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *NetRpcErr) UnmarshalCBOR(r io.Reader) (err error) {
|
||||
*t = NetRpcErr{}
|
||||
|
||||
cr := cbg.NewCborReader(r)
|
||||
|
||||
maj, extra, err := cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
}()
|
||||
|
||||
if maj != cbg.MajArray {
|
||||
return fmt.Errorf("cbor input should be of type array")
|
||||
}
|
||||
|
||||
if extra != 3 {
|
||||
return fmt.Errorf("cbor input had wrong number of fields")
|
||||
}
|
||||
|
||||
// t.Type (blockstore.NetRPCErrType) (uint8)
|
||||
|
||||
maj, extra, err = cr.ReadHeader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if maj != cbg.MajUnsignedInt {
|
||||
return fmt.Errorf("wrong type for uint8 field")
|
||||
}
|
||||
if extra > math.MaxUint8 {
|
||||
return fmt.Errorf("integer in input was too large for uint8 field")
|
||||
}
|
||||
t.Type = NetRPCErrType(extra)
|
||||
// t.Msg (string) (string)
|
||||
|
||||
{
|
||||
sval, err := cbg.ReadString(cr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.Msg = string(sval)
|
||||
}
|
||||
// t.Cid (cid.Cid) (struct)
|
||||
|
||||
{
|
||||
|
||||
b, err := cr.ReadByte()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b != cbg.CborNull[0] {
|
||||
if err := cr.UnreadByte(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c, err := cbg.ReadCid(cr)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to read cid field t.Cid: %w", err)
|
||||
}
|
||||
|
||||
t.Cid = &c
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type hotViewKey struct{}
|
||||
|
||||
var hotView = hotViewKey{}
|
||||
|
||||
// WithHotView constructs a new context with an option that provides a hint to the blockstore
|
||||
// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
|
||||
func WithHotView(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, hotView, struct{}{})
|
||||
}
|
||||
|
||||
// IsHotView returns true if the hot view option is set in the context
|
||||
func IsHotView(ctx context.Context) bool {
|
||||
v := ctx.Value(hotView)
|
||||
return v != nil
|
||||
}
|
||||
+13
-17
@@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
var _ Blockstore = (*discardstore)(nil)
|
||||
@@ -18,43 +18,39 @@ func NewDiscardStore(bs Blockstore) Blockstore {
|
||||
return &discardstore{bs: bs}
|
||||
}
|
||||
|
||||
func (b *discardstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
return b.bs.Has(ctx, cid)
|
||||
func (b *discardstore) Has(cid cid.Cid) (bool, error) {
|
||||
return b.bs.Has(cid)
|
||||
}
|
||||
|
||||
func (b *discardstore) HashOnRead(hor bool) {
|
||||
b.bs.HashOnRead(hor)
|
||||
}
|
||||
|
||||
func (b *discardstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
return b.bs.Get(ctx, cid)
|
||||
func (b *discardstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
return b.bs.Get(cid)
|
||||
}
|
||||
|
||||
func (b *discardstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
return b.bs.GetSize(ctx, cid)
|
||||
func (b *discardstore) GetSize(cid cid.Cid) (int, error) {
|
||||
return b.bs.GetSize(cid)
|
||||
}
|
||||
|
||||
func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
|
||||
return b.bs.View(ctx, cid, f)
|
||||
func (b *discardstore) View(cid cid.Cid, f func([]byte) error) error {
|
||||
return b.bs.View(cid, f)
|
||||
}
|
||||
|
||||
func (b *discardstore) Flush(ctx context.Context) error {
|
||||
func (b *discardstore) Put(blk blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
func (b *discardstore) PutMany(blks []blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *discardstore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
func (b *discardstore) DeleteBlock(cid cid.Cid) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *discardstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *discardstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
func (b *discardstore) DeleteMany(cids []cid.Cid) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+14
-14
@@ -5,10 +5,10 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
|
||||
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
|
||||
|
||||
if fbs.missFn == nil {
|
||||
log.Errorw("fallbackstore: missFn not configured yet")
|
||||
return nil, ipld.ErrNotFound{Cid: c}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,30 +71,30 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
|
||||
// chain bitswap puts blocks in temp blockstore which is cleaned up
|
||||
// every few min (to drop any messages we fetched but don't want)
|
||||
// in this case we want to keep this block around
|
||||
if err := fbs.Put(ctx, b); err != nil {
|
||||
if err := fbs.Put(b); err != nil {
|
||||
return nil, xerrors.Errorf("persisting fallback-fetched block: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
|
||||
b, err := fbs.Blockstore.Get(ctx, c)
|
||||
switch {
|
||||
case err == nil:
|
||||
func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
b, err := fbs.Blockstore.Get(c)
|
||||
switch err {
|
||||
case nil:
|
||||
return b, nil
|
||||
case ipld.IsNotFound(err):
|
||||
case ErrNotFound:
|
||||
return fbs.getFallback(c)
|
||||
default:
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
|
||||
func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
sz, err := fbs.Blockstore.GetSize(ctx, c)
|
||||
switch {
|
||||
case err == nil:
|
||||
func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error) {
|
||||
sz, err := fbs.Blockstore.GetSize(c)
|
||||
switch err {
|
||||
case nil:
|
||||
return sz, nil
|
||||
case ipld.IsNotFound(err):
|
||||
case ErrNotFound:
|
||||
b, err := fbs.getFallback(c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
||||
+20
-45
@@ -4,10 +4,11 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
var _ Blockstore = (*idstore)(nil)
|
||||
@@ -37,7 +38,7 @@ func decodeCid(cid cid.Cid) (inline bool, data []byte, err error) {
|
||||
return false, nil, err
|
||||
}
|
||||
|
||||
func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
func (b *idstore) Has(cid cid.Cid) (bool, error) {
|
||||
inline, _, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -47,10 +48,10 @@ func (b *idstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return b.bs.Has(ctx, cid)
|
||||
return b.bs.Has(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
func (b *idstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -60,10 +61,10 @@ func (b *idstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
return blocks.NewBlockWithCid(data, cid)
|
||||
}
|
||||
|
||||
return b.bs.Get(ctx, cid)
|
||||
return b.bs.Get(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
func (b *idstore) GetSize(cid cid.Cid) (int, error) {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -73,10 +74,10 @@ func (b *idstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
return len(data), err
|
||||
}
|
||||
|
||||
return b.bs.GetSize(ctx, cid)
|
||||
return b.bs.GetSize(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
|
||||
func (b *idstore) View(cid cid.Cid, cb func([]byte) error) error {
|
||||
inline, data, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -86,10 +87,10 @@ func (b *idstore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error)
|
||||
return cb(data)
|
||||
}
|
||||
|
||||
return b.bs.View(ctx, cid, cb)
|
||||
return b.bs.View(cid, cb)
|
||||
}
|
||||
|
||||
func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
func (b *idstore) Put(blk blocks.Block) error {
|
||||
inline, _, err := decodeCid(blk.Cid())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -99,18 +100,10 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.bs.Put(ctx, blk)
|
||||
return b.bs.Put(blk)
|
||||
}
|
||||
|
||||
func (b *idstore) ForEachKey(f func(cid.Cid) error) error {
|
||||
iterBstore, ok := b.bs.(BlockstoreIterator)
|
||||
if !ok {
|
||||
return xerrors.Errorf("underlying blockstore (type %T) doesn't support fast iteration", b.bs)
|
||||
}
|
||||
return iterBstore.ForEachKey(f)
|
||||
}
|
||||
|
||||
func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
func (b *idstore) PutMany(blks []blocks.Block) error {
|
||||
toPut := make([]blocks.Block, 0, len(blks))
|
||||
for _, blk := range blks {
|
||||
inline, _, err := decodeCid(blk.Cid())
|
||||
@@ -125,13 +118,13 @@ func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
}
|
||||
|
||||
if len(toPut) > 0 {
|
||||
return b.bs.PutMany(ctx, toPut)
|
||||
return b.bs.PutMany(toPut)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
func (b *idstore) DeleteBlock(cid cid.Cid) error {
|
||||
inline, _, err := decodeCid(cid)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error decoding Cid: %w", err)
|
||||
@@ -141,10 +134,10 @@ func (b *idstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.bs.DeleteBlock(ctx, cid)
|
||||
return b.bs.DeleteBlock(cid)
|
||||
}
|
||||
|
||||
func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
func (b *idstore) DeleteMany(cids []cid.Cid) error {
|
||||
toDelete := make([]cid.Cid, 0, len(cids))
|
||||
for _, cid := range cids {
|
||||
inline, _, err := decodeCid(cid)
|
||||
@@ -159,7 +152,7 @@ func (b *idstore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
}
|
||||
|
||||
if len(toDelete) > 0 {
|
||||
return b.bs.DeleteMany(ctx, toDelete)
|
||||
return b.bs.DeleteMany(toDelete)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -179,21 +172,3 @@ func (b *idstore) Close() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *idstore) Flush(ctx context.Context) error {
|
||||
return b.bs.Flush(ctx)
|
||||
}
|
||||
|
||||
func (b *idstore) CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error {
|
||||
if bs, ok := b.bs.(BlockstoreGC); ok {
|
||||
return bs.CollectGarbage(ctx, options...)
|
||||
}
|
||||
return xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
func (b *idstore) GCOnce(ctx context.Context, options ...BlockstoreGCOption) error {
|
||||
if bs, ok := b.bs.(BlockstoreGCOnce); ok {
|
||||
return bs.GCOnce(ctx, options...)
|
||||
}
|
||||
return xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
+25
-24
@@ -3,18 +3,19 @@ package blockstore
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
iface "github.com/ipfs/boxo/coreiface"
|
||||
"github.com/ipfs/boxo/coreiface/options"
|
||||
"github.com/ipfs/boxo/coreiface/path"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
rpc "github.com/filecoin-project/kubo-api-client"
|
||||
"github.com/multiformats/go-multiaddr"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
iface "github.com/ipfs/interface-go-ipfs-core"
|
||||
"github.com/ipfs/interface-go-ipfs-core/options"
|
||||
"github.com/ipfs/interface-go-ipfs-core/path"
|
||||
)
|
||||
|
||||
type IPFSBlockstore struct {
|
||||
@@ -25,7 +26,7 @@ type IPFSBlockstore struct {
|
||||
var _ BasicBlockstore = (*IPFSBlockstore)(nil)
|
||||
|
||||
func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
|
||||
localApi, err := rpc.NewLocalApi()
|
||||
localApi, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("getting local ipfs api: %w", err)
|
||||
}
|
||||
@@ -52,7 +53,7 @@ func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, e
|
||||
}
|
||||
|
||||
func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
|
||||
httpApi, err := rpc.NewApi(maddr)
|
||||
httpApi, err := httpapi.NewApi(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
|
||||
}
|
||||
@@ -78,12 +79,12 @@ func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onl
|
||||
return Adapt(bs), nil
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
|
||||
func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error {
|
||||
return xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
_, err := i.offlineAPI.Block().Stat(ctx, path.IpldPath(cid))
|
||||
func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error) {
|
||||
_, err := i.offlineAPI.Block().Stat(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
// The underlying client is running in Offline mode.
|
||||
// Stat() will fail with an err if the block isn't in the
|
||||
@@ -98,13 +99,13 @@ func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
rd, err := i.api.Block().Get(ctx, path.IpldPath(cid))
|
||||
func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
rd, err := i.api.Block().Get(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("getting ipfs block: %w", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(rd)
|
||||
data, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -112,8 +113,8 @@ func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, er
|
||||
return blocks.NewBlockWithCid(data, cid)
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
st, err := i.api.Block().Stat(ctx, path.IpldPath(cid))
|
||||
func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error) {
|
||||
st, err := i.api.Block().Stat(i.ctx, path.IpldPath(cid))
|
||||
if err != nil {
|
||||
return 0, xerrors.Errorf("getting ipfs block: %w", err)
|
||||
}
|
||||
@@ -121,23 +122,23 @@ func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error)
|
||||
return st.Size(), nil
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
|
||||
func (i *IPFSBlockstore) Put(block blocks.Block) error {
|
||||
mhd, err := multihash.Decode(block.Cid().Hash())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
|
||||
_, err = i.api.Block().Put(i.ctx, bytes.NewReader(block.RawData()),
|
||||
options.Block.Hash(mhd.Code, mhd.Length),
|
||||
options.Block.Format(multihash.Codes[block.Cid().Type()]))
|
||||
options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
|
||||
return err
|
||||
}
|
||||
|
||||
func (i *IPFSBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error {
|
||||
// TODO: could be done in parallel
|
||||
|
||||
for _, block := range blocks {
|
||||
if err := i.Put(ctx, block); err != nil {
|
||||
if err := i.Put(block); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
+23
-30
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
)
|
||||
|
||||
// NewMemory returns a temporary memory-backed blockstore.
|
||||
@@ -14,61 +13,55 @@ func NewMemory() MemBlockstore {
|
||||
}
|
||||
|
||||
// MemBlockstore is a terminal blockstore that keeps blocks in memory.
|
||||
// To match behavior of badger blockstore we index by multihash only.
|
||||
type MemBlockstore map[string]blocks.Block
|
||||
type MemBlockstore map[cid.Cid]blocks.Block
|
||||
|
||||
func (MemBlockstore) Flush(context.Context) error { return nil }
|
||||
|
||||
func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
|
||||
delete(m, string(k.Hash()))
|
||||
func (m MemBlockstore) DeleteBlock(k cid.Cid) error {
|
||||
delete(m, k)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
|
||||
func (m MemBlockstore) DeleteMany(ks []cid.Cid) error {
|
||||
for _, k := range ks {
|
||||
delete(m, string(k.Hash()))
|
||||
delete(m, k)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
|
||||
_, ok := m[string(k.Hash())]
|
||||
func (m MemBlockstore) Has(k cid.Cid) (bool, error) {
|
||||
_, ok := m[k]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
|
||||
b, ok := m[string(k.Hash())]
|
||||
func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return ipld.ErrNotFound{Cid: k}
|
||||
return ErrNotFound
|
||||
}
|
||||
return callback(b.RawData())
|
||||
}
|
||||
|
||||
func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
|
||||
b, ok := m[string(k.Hash())]
|
||||
func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return nil, ipld.ErrNotFound{Cid: k}
|
||||
}
|
||||
if b.Cid().Prefix().Codec != k.Prefix().Codec {
|
||||
return blocks.NewBlockWithCid(b.RawData(), k)
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// GetSize returns the CIDs mapped BlockSize
|
||||
func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
|
||||
b, ok := m[string(k.Hash())]
|
||||
func (m MemBlockstore) GetSize(k cid.Cid) (int, error) {
|
||||
b, ok := m[k]
|
||||
if !ok {
|
||||
return 0, ipld.ErrNotFound{Cid: k}
|
||||
return 0, ErrNotFound
|
||||
}
|
||||
return len(b.RawData()), nil
|
||||
}
|
||||
|
||||
// Put puts a given block to the underlying datastore
|
||||
func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
|
||||
func (m MemBlockstore) Put(b blocks.Block) error {
|
||||
// Convert to a basic block for safety, but try to reuse the existing
|
||||
// block if it's already a basic block.
|
||||
k := string(b.Cid().Hash())
|
||||
k := b.Cid()
|
||||
if _, ok := b.(*blocks.BasicBlock); !ok {
|
||||
// If we already have the block, abort.
|
||||
if _, ok := m[k]; ok {
|
||||
@@ -77,15 +70,15 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
|
||||
// the error is only for debugging.
|
||||
b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
|
||||
}
|
||||
m[k] = b
|
||||
m[b.Cid()] = b
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutMany puts a slice of blocks at the same time using batching
|
||||
// capabilities of the underlying datastore whenever possible.
|
||||
func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
|
||||
func (m MemBlockstore) PutMany(bs []blocks.Block) error {
|
||||
for _, b := range bs {
|
||||
_ = m.Put(ctx, b) // can't fail
|
||||
_ = m.Put(b) // can't fail
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -95,8 +88,8 @@ func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
|
||||
// the given context, closing the channel if it becomes Done.
|
||||
func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
ch := make(chan cid.Cid, len(m))
|
||||
for _, b := range m {
|
||||
ch <- b.Cid()
|
||||
for k := range m {
|
||||
ch <- k
|
||||
}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMemGetCodec(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
bs := NewMemory()
|
||||
|
||||
cborArr := []byte{0x82, 1, 2}
|
||||
|
||||
h, err := mh.Sum(cborArr, mh.SHA2_256, -1)
|
||||
require.NoError(t, err)
|
||||
|
||||
rawCid := cid.NewCidV1(cid.Raw, h)
|
||||
rawBlk, err := blocks.NewBlockWithCid(cborArr, rawCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = bs.Put(ctx, rawBlk)
|
||||
require.NoError(t, err)
|
||||
|
||||
cborCid := cid.NewCidV1(cid.DagCBOR, h)
|
||||
|
||||
cborBlk, err := bs.Get(ctx, cborCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, cborCid.Prefix(), cborBlk.Cid().Prefix())
|
||||
require.EqualValues(t, cborArr, cborBlk.RawData())
|
||||
|
||||
// was allocated
|
||||
require.NotEqual(t, cborBlk, rawBlk)
|
||||
|
||||
gotRawBlk, err := bs.Get(ctx, rawCid)
|
||||
require.NoError(t, err)
|
||||
|
||||
// not allocated
|
||||
require.Equal(t, rawBlk, gotRawBlk)
|
||||
}
|
||||
@@ -1,426 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/libp2p/go-msgio"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type NetRPCReqType byte
|
||||
|
||||
const (
|
||||
NRpcHas NetRPCReqType = iota
|
||||
NRpcGet
|
||||
NRpcGetSize
|
||||
NRpcPut
|
||||
NRpcDelete
|
||||
|
||||
// todo cancel req
|
||||
)
|
||||
|
||||
type NetRPCRespType byte
|
||||
|
||||
const (
|
||||
NRpcOK NetRPCRespType = iota
|
||||
NRpcErr
|
||||
NRpcMore
|
||||
)
|
||||
|
||||
type NetRPCErrType byte
|
||||
|
||||
const (
|
||||
NRpcErrGeneric NetRPCErrType = iota
|
||||
NRpcErrNotFound
|
||||
)
|
||||
|
||||
type NetRpcReq struct {
|
||||
Type NetRPCReqType
|
||||
ID uint64
|
||||
|
||||
Cid []cid.Cid // todo maxsize?
|
||||
Data [][]byte // todo maxsize?
|
||||
}
|
||||
|
||||
type NetRpcResp struct {
|
||||
Type NetRPCRespType
|
||||
ID uint64
|
||||
|
||||
// error or cids in allkeys
|
||||
Data []byte // todo maxsize?
|
||||
|
||||
next <-chan NetRpcResp
|
||||
}
|
||||
|
||||
type NetRpcErr struct {
|
||||
Type NetRPCErrType
|
||||
|
||||
Msg string
|
||||
|
||||
// in case of NRpcErrNotFound
|
||||
Cid *cid.Cid
|
||||
}
|
||||
|
||||
type NetworkStore struct {
|
||||
// note: writer is thread-safe
|
||||
msgStream msgio.ReadWriteCloser
|
||||
|
||||
// atomic
|
||||
reqCount uint64
|
||||
|
||||
respLk sync.Mutex
|
||||
|
||||
// respMap is nil after store closes
|
||||
respMap map[uint64]chan<- NetRpcResp
|
||||
|
||||
closing chan struct{}
|
||||
closed chan struct{}
|
||||
|
||||
closeLk sync.Mutex
|
||||
onClose []func()
|
||||
}
|
||||
|
||||
func NewNetworkStore(mss msgio.ReadWriteCloser) *NetworkStore {
|
||||
ns := &NetworkStore{
|
||||
msgStream: mss,
|
||||
|
||||
respMap: map[uint64]chan<- NetRpcResp{},
|
||||
|
||||
closing: make(chan struct{}),
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
|
||||
go ns.receive()
|
||||
|
||||
return ns
|
||||
}
|
||||
|
||||
func (n *NetworkStore) shutdown(msg string) {
|
||||
if err := n.msgStream.Close(); err != nil {
|
||||
log.Errorw("closing netstore msg stream", "error", err)
|
||||
}
|
||||
|
||||
nerr := NetRpcErr{
|
||||
Type: NRpcErrGeneric,
|
||||
Msg: msg,
|
||||
Cid: nil,
|
||||
}
|
||||
|
||||
var errb bytes.Buffer
|
||||
if err := nerr.MarshalCBOR(&errb); err != nil {
|
||||
log.Errorw("netstore shutdown: error marshaling error", "err", err)
|
||||
}
|
||||
|
||||
n.respLk.Lock()
|
||||
for id, resps := range n.respMap {
|
||||
resps <- NetRpcResp{
|
||||
Type: NRpcErr,
|
||||
ID: id,
|
||||
Data: errb.Bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
n.respMap = nil
|
||||
|
||||
n.respLk.Unlock()
|
||||
}
|
||||
|
||||
func (n *NetworkStore) OnClose(cb func()) {
|
||||
n.closeLk.Lock()
|
||||
defer n.closeLk.Unlock()
|
||||
|
||||
select {
|
||||
case <-n.closed:
|
||||
cb()
|
||||
default:
|
||||
n.onClose = append(n.onClose, cb)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NetworkStore) receive() {
|
||||
defer func() {
|
||||
n.closeLk.Lock()
|
||||
defer n.closeLk.Unlock()
|
||||
|
||||
close(n.closed)
|
||||
if n.onClose != nil {
|
||||
for _, f := range n.onClose {
|
||||
f()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-n.closing:
|
||||
n.shutdown("netstore stopping")
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
msg, err := n.msgStream.ReadMsg()
|
||||
if err != nil {
|
||||
n.shutdown(fmt.Sprintf("netstore ReadMsg: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
var resp NetRpcResp
|
||||
if err := resp.UnmarshalCBOR(bytes.NewReader(msg)); err != nil {
|
||||
n.shutdown(fmt.Sprintf("unmarshaling netstore response: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
n.msgStream.ReleaseMsg(msg)
|
||||
|
||||
n.respLk.Lock()
|
||||
if ch, ok := n.respMap[resp.ID]; ok {
|
||||
if resp.Type == NRpcMore {
|
||||
nch := make(chan NetRpcResp, 1)
|
||||
resp.next = nch
|
||||
n.respMap[resp.ID] = nch
|
||||
} else {
|
||||
delete(n.respMap, resp.ID)
|
||||
}
|
||||
|
||||
ch <- resp
|
||||
}
|
||||
n.respLk.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NetworkStore) sendRpc(rt NetRPCReqType, cids []cid.Cid, data [][]byte) (uint64, <-chan NetRpcResp, error) {
|
||||
rid := atomic.AddUint64(&n.reqCount, 1)
|
||||
|
||||
respCh := make(chan NetRpcResp, 1) // todo pool?
|
||||
|
||||
n.respLk.Lock()
|
||||
if n.respMap == nil {
|
||||
n.respLk.Unlock()
|
||||
return 0, nil, xerrors.Errorf("netstore closed")
|
||||
}
|
||||
n.respMap[rid] = respCh
|
||||
n.respLk.Unlock()
|
||||
|
||||
req := NetRpcReq{
|
||||
Type: rt,
|
||||
ID: rid,
|
||||
Cid: cids,
|
||||
Data: data,
|
||||
}
|
||||
|
||||
var rbuf bytes.Buffer // todo buffer pool
|
||||
if err := req.MarshalCBOR(&rbuf); err != nil {
|
||||
n.respLk.Lock()
|
||||
defer n.respLk.Unlock()
|
||||
|
||||
if n.respMap == nil {
|
||||
return 0, nil, xerrors.Errorf("netstore closed")
|
||||
}
|
||||
delete(n.respMap, rid)
|
||||
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
if err := n.msgStream.WriteMsg(rbuf.Bytes()); err != nil {
|
||||
n.respLk.Lock()
|
||||
defer n.respLk.Unlock()
|
||||
|
||||
if n.respMap == nil {
|
||||
return 0, nil, xerrors.Errorf("netstore closed")
|
||||
}
|
||||
delete(n.respMap, rid)
|
||||
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
return rid, respCh, nil
|
||||
}
|
||||
|
||||
func (n *NetworkStore) waitResp(ctx context.Context, rch <-chan NetRpcResp, rid uint64) (NetRpcResp, error) {
|
||||
select {
|
||||
case resp := <-rch:
|
||||
if resp.Type == NRpcErr {
|
||||
var e NetRpcErr
|
||||
if err := e.UnmarshalCBOR(bytes.NewReader(resp.Data)); err != nil {
|
||||
return NetRpcResp{}, xerrors.Errorf("unmarshaling error data: %w", err)
|
||||
}
|
||||
|
||||
var err error
|
||||
switch e.Type {
|
||||
case NRpcErrNotFound:
|
||||
if e.Cid != nil {
|
||||
err = ipld.ErrNotFound{
|
||||
Cid: *e.Cid,
|
||||
}
|
||||
} else {
|
||||
err = xerrors.Errorf("block not found, but cid was null")
|
||||
}
|
||||
case NRpcErrGeneric:
|
||||
err = xerrors.Errorf("generic error")
|
||||
default:
|
||||
err = xerrors.Errorf("unknown error type")
|
||||
}
|
||||
|
||||
return NetRpcResp{}, xerrors.Errorf("netstore error response: %s (%w)", e.Msg, err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
case <-ctx.Done():
|
||||
// todo send cancel req
|
||||
|
||||
n.respLk.Lock()
|
||||
if n.respMap != nil {
|
||||
delete(n.respMap, rid)
|
||||
}
|
||||
n.respLk.Unlock()
|
||||
|
||||
return NetRpcResp{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NetworkStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
req, rch, err := n.sendRpc(NRpcHas, []cid.Cid{c}, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
resp, err := n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(resp.Data) != 1 {
|
||||
return false, xerrors.Errorf("expected reposnse length to be 1 byte")
|
||||
}
|
||||
switch resp.Data[0] {
|
||||
case cbg.CborBoolTrue[0]:
|
||||
return true, nil
|
||||
case cbg.CborBoolFalse[0]:
|
||||
return false, nil
|
||||
default:
|
||||
return false, xerrors.Errorf("has: bad response: %x", resp.Data[0])
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NetworkStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
|
||||
req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blocks.NewBlockWithCid(resp.Data, c)
|
||||
}
|
||||
|
||||
func (n *NetworkStore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
|
||||
req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return callback(resp.Data) // todo return buf to pool
|
||||
}
|
||||
|
||||
func (n *NetworkStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
req, rch, err := n.sendRpc(NRpcGetSize, []cid.Cid{c}, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
resp, err := n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if len(resp.Data) != 4 {
|
||||
return 0, xerrors.Errorf("expected getsize response to be 4 bytes, was %d", resp.Data)
|
||||
}
|
||||
|
||||
return int(binary.LittleEndian.Uint32(resp.Data)), nil
|
||||
}
|
||||
|
||||
func (n *NetworkStore) Put(ctx context.Context, block blocks.Block) error {
|
||||
return n.PutMany(ctx, []blocks.Block{block})
|
||||
}
|
||||
|
||||
func (n *NetworkStore) PutMany(ctx context.Context, blocks []blocks.Block) error {
|
||||
// todo pool
|
||||
cids := make([]cid.Cid, len(blocks))
|
||||
blkDatas := make([][]byte, len(blocks))
|
||||
for i, block := range blocks {
|
||||
cids[i] = block.Cid()
|
||||
blkDatas[i] = block.RawData()
|
||||
}
|
||||
|
||||
req, rch, err := n.sendRpc(NRpcPut, cids, blkDatas)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NetworkStore) DeleteBlock(ctx context.Context, c cid.Cid) error {
|
||||
return n.DeleteMany(ctx, []cid.Cid{c})
|
||||
}
|
||||
|
||||
func (n *NetworkStore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
|
||||
req, rch, err := n.sendRpc(NRpcDelete, cids, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = n.waitResp(ctx, rch, req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NetworkStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, xerrors.Errorf("not supported")
|
||||
}
|
||||
|
||||
func (n *NetworkStore) HashOnRead(enabled bool) {
|
||||
// todo
|
||||
return
|
||||
}
|
||||
|
||||
func (*NetworkStore) Flush(context.Context) error { return nil }
|
||||
|
||||
func (n *NetworkStore) Stop(ctx context.Context) error {
|
||||
close(n.closing)
|
||||
|
||||
select {
|
||||
case <-n.closed:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
var _ Blockstore = &NetworkStore{}
|
||||
@@ -1,237 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/libp2p/go-msgio"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type NetworkStoreHandler struct {
|
||||
msgStream msgio.ReadWriteCloser
|
||||
|
||||
bs Blockstore
|
||||
}
|
||||
|
||||
// NOTE: This code isn't yet hardened to accept untrusted input. See TODOs here and in net.go
|
||||
func HandleNetBstoreStream(ctx context.Context, bs Blockstore, mss msgio.ReadWriteCloser) *NetworkStoreHandler {
|
||||
ns := &NetworkStoreHandler{
|
||||
msgStream: mss,
|
||||
bs: bs,
|
||||
}
|
||||
|
||||
go ns.handle(ctx)
|
||||
|
||||
return ns
|
||||
}
|
||||
|
||||
func (h *NetworkStoreHandler) handle(ctx context.Context) {
|
||||
defer func() {
|
||||
if err := h.msgStream.Close(); err != nil {
|
||||
log.Errorw("error closing blockstore stream", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
var req NetRpcReq
|
||||
|
||||
ms, err := h.msgStream.ReadMsg()
|
||||
if err != nil {
|
||||
log.Warnw("bstore stream err", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := req.UnmarshalCBOR(bytes.NewReader(ms)); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.msgStream.ReleaseMsg(ms)
|
||||
|
||||
switch req.Type {
|
||||
case NRpcHas:
|
||||
if len(req.Cid) != 1 {
|
||||
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := h.bs.Has(ctx, req.Cid[0])
|
||||
if err != nil {
|
||||
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var resData [1]byte
|
||||
if res {
|
||||
resData[0] = cbg.CborBoolTrue[0]
|
||||
} else {
|
||||
resData[0] = cbg.CborBoolFalse[0]
|
||||
}
|
||||
|
||||
if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
|
||||
log.Warnw("writing response", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
case NRpcGet:
|
||||
if len(req.Cid) != 1 {
|
||||
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
err := h.bs.View(ctx, req.Cid[0], func(bdata []byte) error {
|
||||
return h.respond(req.ID, NRpcOK, bdata)
|
||||
})
|
||||
if err != nil {
|
||||
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
case NRpcGetSize:
|
||||
if len(req.Cid) != 1 {
|
||||
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
sz, err := h.bs.GetSize(ctx, req.Cid[0])
|
||||
if err != nil {
|
||||
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var resData [4]byte
|
||||
binary.LittleEndian.PutUint32(resData[:], uint32(sz))
|
||||
|
||||
if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
|
||||
log.Warnw("writing response", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
case NRpcPut:
|
||||
blocks := make([]block.Block, len(req.Cid))
|
||||
|
||||
if len(req.Cid) != len(req.Data) {
|
||||
if err := h.respondError(req.ID, xerrors.New("cid count didn't match data count"), cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for i := range req.Cid {
|
||||
blocks[i], err = block.NewBlockWithCid(req.Data[i], req.Cid[i])
|
||||
if err != nil {
|
||||
log.Warnw("make block", "error", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err := h.bs.PutMany(ctx, blocks)
|
||||
if err != nil {
|
||||
if err := h.respondError(req.ID, err, cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
|
||||
log.Warnw("writing response", "error", err)
|
||||
return
|
||||
}
|
||||
case NRpcDelete:
|
||||
err := h.bs.DeleteMany(ctx, req.Cid)
|
||||
if err != nil {
|
||||
if err := h.respondError(req.ID, err, cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
|
||||
log.Warnw("writing response", "error", err)
|
||||
return
|
||||
}
|
||||
default:
|
||||
if err := h.respondError(req.ID, xerrors.New("unsupported request type"), cid.Undef); err != nil {
|
||||
log.Warnw("writing error response", "error", err)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *NetworkStoreHandler) respondError(req uint64, uerr error, c cid.Cid) error {
|
||||
var resp NetRpcResp
|
||||
resp.ID = req
|
||||
resp.Type = NRpcErr
|
||||
|
||||
nerr := NetRpcErr{
|
||||
Type: NRpcErrGeneric,
|
||||
Msg: uerr.Error(),
|
||||
}
|
||||
if ipld.IsNotFound(uerr) {
|
||||
nerr.Type = NRpcErrNotFound
|
||||
nerr.Cid = &c
|
||||
}
|
||||
|
||||
var edata bytes.Buffer
|
||||
if err := nerr.MarshalCBOR(&edata); err != nil {
|
||||
return xerrors.Errorf("marshaling error data: %w", err)
|
||||
}
|
||||
|
||||
resp.Data = edata.Bytes()
|
||||
|
||||
var msg bytes.Buffer
|
||||
if err := resp.MarshalCBOR(&msg); err != nil {
|
||||
return xerrors.Errorf("marshaling error response: %w", err)
|
||||
}
|
||||
|
||||
if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
|
||||
return xerrors.Errorf("write error response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *NetworkStoreHandler) respond(req uint64, rt NetRPCRespType, data []byte) error {
|
||||
var resp NetRpcResp
|
||||
resp.ID = req
|
||||
resp.Type = rt
|
||||
resp.Data = data
|
||||
|
||||
var msg bytes.Buffer
|
||||
if err := resp.MarshalCBOR(&msg); err != nil {
|
||||
return xerrors.Errorf("marshaling response: %w", err)
|
||||
}
|
||||
|
||||
if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
|
||||
return xerrors.Errorf("write response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
block "github.com/ipfs/go-block-format"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"github.com/libp2p/go-msgio"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNetBstore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
cr, sw := io.Pipe()
|
||||
sr, cw := io.Pipe()
|
||||
|
||||
cm := msgio.Combine(msgio.NewWriter(cw), msgio.NewReader(cr))
|
||||
sm := msgio.Combine(msgio.NewWriter(sw), msgio.NewReader(sr))
|
||||
|
||||
bbs := NewMemorySync()
|
||||
_ = HandleNetBstoreStream(ctx, bbs, sm)
|
||||
|
||||
nbs := NewNetworkStore(cm)
|
||||
|
||||
tb1 := block.NewBlock([]byte("aoeu"))
|
||||
|
||||
h, err := nbs.Has(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.False(t, h)
|
||||
|
||||
err = nbs.Put(ctx, tb1)
|
||||
require.NoError(t, err)
|
||||
|
||||
h, err = nbs.Has(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.True(t, h)
|
||||
|
||||
sz, err := nbs.GetSize(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, sz)
|
||||
|
||||
err = nbs.DeleteBlock(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
|
||||
h, err = nbs.Has(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.False(t, h)
|
||||
|
||||
_, err = nbs.Get(ctx, tb1.Cid())
|
||||
fmt.Println(err)
|
||||
require.True(t, ipld.IsNotFound(err))
|
||||
|
||||
err = nbs.Put(ctx, tb1)
|
||||
require.NoError(t, err)
|
||||
|
||||
b, err := nbs.Get(ctx, tb1.Cid())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "aoeu", string(b.RawData()))
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package blockstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/libp2p/go-msgio"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type wsWrapper struct {
|
||||
wc *websocket.Conn
|
||||
|
||||
nextMsg []byte
|
||||
}
|
||||
|
||||
func (w *wsWrapper) Read(b []byte) (int, error) {
|
||||
return 0, xerrors.New("read unsupported")
|
||||
}
|
||||
|
||||
func (w *wsWrapper) ReadMsg() ([]byte, error) {
|
||||
if w.nextMsg != nil {
|
||||
nm := w.nextMsg
|
||||
w.nextMsg = nil
|
||||
return nm, nil
|
||||
}
|
||||
|
||||
mt, r, err := w.wc.NextReader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch mt {
|
||||
case websocket.BinaryMessage, websocket.TextMessage:
|
||||
default:
|
||||
return nil, xerrors.Errorf("unexpected message type")
|
||||
}
|
||||
|
||||
// todo pool
|
||||
// todo limit sizes
|
||||
var mbuf bytes.Buffer
|
||||
if _, err := mbuf.ReadFrom(r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return mbuf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (w *wsWrapper) ReleaseMsg(bytes []byte) {
|
||||
// todo use a pool
|
||||
}
|
||||
|
||||
func (w *wsWrapper) NextMsgLen() (int, error) {
|
||||
if w.nextMsg != nil {
|
||||
return len(w.nextMsg), nil
|
||||
}
|
||||
|
||||
mt, msg, err := w.wc.ReadMessage()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
switch mt {
|
||||
case websocket.BinaryMessage, websocket.TextMessage:
|
||||
default:
|
||||
return 0, xerrors.Errorf("unexpected message type")
|
||||
}
|
||||
|
||||
w.nextMsg = msg
|
||||
return len(w.nextMsg), nil
|
||||
}
|
||||
|
||||
func (w *wsWrapper) Write(bytes []byte) (int, error) {
|
||||
return 0, xerrors.New("write unsupported")
|
||||
}
|
||||
|
||||
func (w *wsWrapper) WriteMsg(bytes []byte) error {
|
||||
return w.wc.WriteMessage(websocket.BinaryMessage, bytes)
|
||||
}
|
||||
|
||||
func (w *wsWrapper) Close() error {
|
||||
return w.wc.Close()
|
||||
}
|
||||
|
||||
var _ msgio.ReadWriteCloser = &wsWrapper{}
|
||||
|
||||
func wsConnToMio(wc *websocket.Conn) msgio.ReadWriteCloser {
|
||||
return &wsWrapper{
|
||||
wc: wc,
|
||||
}
|
||||
}
|
||||
|
||||
func HandleNetBstoreWS(ctx context.Context, bs Blockstore, wc *websocket.Conn) *NetworkStoreHandler {
|
||||
return HandleNetBstoreStream(ctx, bs, wsConnToMio(wc))
|
||||
}
|
||||
|
||||
func NewNetworkStoreWS(wc *websocket.Conn) *NetworkStore {
|
||||
return NewNetworkStore(wsConnToMio(wc))
|
||||
}
|
||||
@@ -49,11 +49,10 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
|
||||
blockstore and discards writes; this is necessary to support syncing from a snapshot.
|
||||
- `MarkSetType` -- specifies the type of markset to use during compaction.
|
||||
The markset is the data structure used by compaction/gc to track live objects.
|
||||
The default value is "badger", which will use a disk backed markset using badger.
|
||||
If you have a lot of memory (48G or more) you can also use "map", which will use
|
||||
an in memory markset, speeding up compaction at the cost of higher memory usage.
|
||||
Note: If you are using a VPS with a network volume, you need to provision at least
|
||||
3000 IOPs with the badger markset.
|
||||
The default value is `"map"`, which will use an in-memory map; if you are limited
|
||||
in memory (or indeed see compaction run out of memory), you can also specify
|
||||
`"badger"` which will use an disk backed markset, using badger. This will use
|
||||
much less memory, but will also make compaction slower.
|
||||
- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
|
||||
finalities maintained by default, to maintain messages and message receipts in the
|
||||
hotstore. This is useful for assistive nodes that want to support syncing for other
|
||||
@@ -106,12 +105,6 @@ Compaction works transactionally with the following algorithm:
|
||||
- We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
|
||||
- We then end the transaction and compact/gc the hotstore.
|
||||
|
||||
As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
|
||||
modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
|
||||
memory usage; in fact, when using badger as the markset compaction uses very little memory, and
|
||||
it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
|
||||
memory during compaction.
|
||||
|
||||
## Garbage Collection
|
||||
|
||||
TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
|
||||
@@ -126,7 +119,6 @@ TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
|
||||
It can also optionally compact/gc the coldstore after the copy (with the `--gc-coldstore` flag)
|
||||
and automatically rewrite the lotus config to disable splitstore (with the `--rewrite-config` flag).
|
||||
Note: the node *must be stopped* before running this command.
|
||||
- `clear` -- clears a splitstore installation for restart from snapshot.
|
||||
- `check` -- asynchronously runs a basic healthcheck on the splitstore.
|
||||
The results are appended to `<lotus-repo>/datastore/splitstore/check.txt`.
|
||||
- `info` -- prints some basic information about the splitstore.
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type Checkpoint struct {
|
||||
file *os.File
|
||||
buf *bufio.Writer
|
||||
}
|
||||
|
||||
func NewCheckpoint(path string) (*Checkpoint, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating checkpoint: %w", err)
|
||||
}
|
||||
buf := bufio.NewWriter(file)
|
||||
|
||||
return &Checkpoint{
|
||||
file: file,
|
||||
buf: buf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
|
||||
filein, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
|
||||
}
|
||||
defer filein.Close() //nolint:errcheck
|
||||
|
||||
bufin := bufio.NewReader(filein)
|
||||
start, err := readRawCid(bufin, nil)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
|
||||
}
|
||||
|
||||
fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
|
||||
if err != nil {
|
||||
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
|
||||
}
|
||||
bufout := bufio.NewWriter(fileout)
|
||||
|
||||
return &Checkpoint{
|
||||
file: fileout,
|
||||
buf: bufout,
|
||||
}, start, nil
|
||||
}
|
||||
|
||||
func (cp *Checkpoint) Set(c cid.Cid) error {
|
||||
if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
|
||||
return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
|
||||
}
|
||||
|
||||
if err := writeRawCid(cp.buf, c, true); err != nil {
|
||||
return xerrors.Errorf("error writing cid to checkpoint: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cp *Checkpoint) Close() error {
|
||||
if cp.file == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := cp.file.Close()
|
||||
cp.file = nil
|
||||
cp.buf = nil
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
|
||||
sz, err := buf.ReadByte()
|
||||
if err != nil {
|
||||
return cid.Undef, err // don't wrap EOF as it is not an error here
|
||||
}
|
||||
|
||||
if hbuf == nil {
|
||||
hbuf = make([]byte, int(sz))
|
||||
} else {
|
||||
hbuf = hbuf[:int(sz)]
|
||||
}
|
||||
|
||||
if _, err := io.ReadFull(buf, hbuf); err != nil {
|
||||
return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
|
||||
}
|
||||
|
||||
hash, err := mh.Cast(hbuf)
|
||||
if err != nil {
|
||||
return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, hash), nil
|
||||
}
|
||||
|
||||
func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
|
||||
hash := c.Hash()
|
||||
if err := buf.WriteByte(byte(len(hash))); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := buf.Write(hash); err != nil {
|
||||
return err
|
||||
}
|
||||
if flush {
|
||||
return buf.Flush()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
func TestCheckpoint(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
path := filepath.Join(dir, "checkpoint")
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
cp, err := NewCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Set(k1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cp.Set(k2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cp, start, err := OpenCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !start.Equals(k2) {
|
||||
t.Fatalf("expected start to be %s; got %s", k2, start)
|
||||
}
|
||||
|
||||
if err := cp.Set(k3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cp.Set(k4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cp, start, err = OpenCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !start.Equals(k4) {
|
||||
t.Fatalf("expected start to be %s; got %s", k4, start)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// also test correct operation with an empty checkpoint
|
||||
cp, err = NewCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cp, start, err = OpenCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if start.Defined() {
|
||||
t.Fatal("expected start to be undefined")
|
||||
}
|
||||
|
||||
if err := cp.Set(k1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cp.Set(k2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cp, start, err = OpenCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !start.Equals(k2) {
|
||||
t.Fatalf("expected start to be %s; got %s", k2, start)
|
||||
}
|
||||
|
||||
if err := cp.Set(k3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cp.Set(k4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cp, start, err = OpenCheckpoint(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !start.Equals(k4) {
|
||||
t.Fatalf("expected start to be %s; got %s", k4, start)
|
||||
}
|
||||
|
||||
if err := cp.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type ColdSetWriter struct {
|
||||
file *os.File
|
||||
buf *bufio.Writer
|
||||
}
|
||||
|
||||
type ColdSetReader struct {
|
||||
file *os.File
|
||||
buf *bufio.Reader
|
||||
}
|
||||
|
||||
func NewColdSetWriter(path string) (*ColdSetWriter, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating coldset: %w", err)
|
||||
}
|
||||
buf := bufio.NewWriter(file)
|
||||
|
||||
return &ColdSetWriter{
|
||||
file: file,
|
||||
buf: buf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewColdSetReader(path string) (*ColdSetReader, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error opening coldset: %w", err)
|
||||
}
|
||||
buf := bufio.NewReader(file)
|
||||
|
||||
return &ColdSetReader{
|
||||
file: file,
|
||||
buf: buf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ColdSetWriter) Write(c cid.Cid) error {
|
||||
return writeRawCid(s.buf, c, false)
|
||||
}
|
||||
|
||||
func (s *ColdSetWriter) Close() error {
|
||||
if s.file == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err1 := s.buf.Flush()
|
||||
err2 := s.file.Close()
|
||||
s.buf = nil
|
||||
s.file = nil
|
||||
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
return err2
|
||||
}
|
||||
|
||||
func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
|
||||
hbuf := make([]byte, 256)
|
||||
for {
|
||||
next, err := readRawCid(s.buf, hbuf)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
|
||||
return xerrors.Errorf("error reading coldset: %w", err)
|
||||
}
|
||||
|
||||
if err := f(next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ColdSetReader) Reset() error {
|
||||
_, err := s.file.Seek(0, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ColdSetReader) Close() error {
|
||||
if s.file == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := s.file.Close()
|
||||
s.file = nil
|
||||
s.buf = nil
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
func TestColdSet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
path := filepath.Join(dir, "coldset")
|
||||
|
||||
makeCid := func(i int) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
const count = 1000
|
||||
cids := make([]cid.Cid, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
cids = append(cids, makeCid(i))
|
||||
}
|
||||
|
||||
cw, err := NewColdSetWriter(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, c := range cids {
|
||||
if err := cw.Write(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cr, err := NewColdSetReader(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
index := 0
|
||||
err = cr.ForEach(func(c cid.Cid) error {
|
||||
if index >= count {
|
||||
t.Fatal("too many cids")
|
||||
}
|
||||
|
||||
if !c.Equals(cids[index]) {
|
||||
t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
|
||||
}
|
||||
|
||||
index++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cr.Reset(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
index = 0
|
||||
err = cr.ForEach(func(c cid.Cid) error {
|
||||
if index >= count {
|
||||
t.Fatal("too many cids")
|
||||
}
|
||||
|
||||
if !c.Equals(cids[index]) {
|
||||
t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
|
||||
}
|
||||
|
||||
index++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -12,10 +12,11 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"go.uber.org/multierr"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type debugLog struct {
|
||||
|
||||
@@ -3,42 +3,35 @@ package splitstore
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
var errMarkSetClosed = errors.New("markset closed")
|
||||
|
||||
// MarkSet is an interface for tracking CIDs during chain and object walks
|
||||
// MarkSet is a utility to keep track of seen CID, and later query for them.
|
||||
//
|
||||
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
|
||||
// * If a probabilistic result is acceptable, it can be backed by a bloom filter
|
||||
type MarkSet interface {
|
||||
ObjectVisitor
|
||||
Mark(cid.Cid) error
|
||||
MarkMany([]cid.Cid) error
|
||||
Has(cid.Cid) (bool, error)
|
||||
Close() error
|
||||
|
||||
// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
|
||||
// of abnormal termination during the critical section span.
|
||||
BeginCriticalSection() error
|
||||
// EndCriticalSection ends the critical section span.
|
||||
EndCriticalSection()
|
||||
SetConcurrent()
|
||||
}
|
||||
|
||||
type MarkSetEnv interface {
|
||||
// New creates a new markset within the environment.
|
||||
// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
|
||||
// sizeHint is a hint about the expected size of the markset
|
||||
New(name string, sizeHint int64) (MarkSet, error)
|
||||
// Recover recovers an existing markset persisted on-disk.
|
||||
Recover(name string) (MarkSet, error)
|
||||
// Close closes the markset
|
||||
Create(name string, sizeHint int64) (MarkSet, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
|
||||
switch mtype {
|
||||
case "bloom":
|
||||
return NewBloomMarkSetEnv()
|
||||
case "map":
|
||||
return NewMapMarkSetEnv(path)
|
||||
return NewMapMarkSetEnv()
|
||||
case "badger":
|
||||
return NewBadgerMarkSetEnv(path)
|
||||
default:
|
||||
|
||||
@@ -3,16 +3,15 @@ package splitstore
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/dgraph-io/badger/v2"
|
||||
"github.com/dgraph-io/badger/v2/options"
|
||||
"github.com/ipfs/go-cid"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/system"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type BadgerMarkSetEnv struct {
|
||||
@@ -28,8 +27,6 @@ type BadgerMarkSet struct {
|
||||
writing map[int]map[string]struct{}
|
||||
writers int
|
||||
seqno int
|
||||
version int
|
||||
persist bool
|
||||
|
||||
db *badger.DB
|
||||
path string
|
||||
@@ -49,12 +46,39 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
|
||||
return &BadgerMarkSetEnv{path: msPath}, nil
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
|
||||
func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
path := filepath.Join(e.path, name)
|
||||
|
||||
db, err := openBadgerDB(path, false)
|
||||
// clean up first
|
||||
err := os.RemoveAll(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating badger db: %w", err)
|
||||
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
|
||||
}
|
||||
|
||||
err = os.MkdirAll(path, 0755) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating markset directory: %w", err)
|
||||
}
|
||||
|
||||
opts := badger.DefaultOptions(path)
|
||||
opts.SyncWrites = false
|
||||
opts.CompactL0OnClose = false
|
||||
opts.Compression = options.None
|
||||
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
|
||||
// between the system blockstore and the markset.
|
||||
// It was observed that using the default memory mapped option resulted in
|
||||
// significant interference and unacceptably high block validation times once the markset
|
||||
// exceeded 1GB in size.
|
||||
opts.TableLoadingMode = options.FileIO
|
||||
opts.ValueLogLoadingMode = options.FileIO
|
||||
opts.Logger = &badgerLogger{
|
||||
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
|
||||
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
|
||||
}
|
||||
|
||||
db, err := badger.Open(opts)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating badger markset: %w", err)
|
||||
}
|
||||
|
||||
ms := &BadgerMarkSet{
|
||||
@@ -68,251 +92,30 @@ func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
|
||||
path := filepath.Join(e.path, name)
|
||||
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return nil, xerrors.Errorf("error stating badger db path: %w", err)
|
||||
}
|
||||
|
||||
db, err := openBadgerDB(path, true)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating badger db: %w", err)
|
||||
}
|
||||
|
||||
ms := &BadgerMarkSet{
|
||||
pend: make(map[string]struct{}),
|
||||
writing: make(map[int]map[string]struct{}),
|
||||
db: db,
|
||||
path: path,
|
||||
persist: true,
|
||||
}
|
||||
ms.cond.L = &ms.mx
|
||||
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
func (e *BadgerMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) BeginCriticalSection() error {
|
||||
s.mx.Lock()
|
||||
|
||||
if s.persist {
|
||||
s.mx.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
var write bool
|
||||
var seqno int
|
||||
if len(s.pend) > 0 {
|
||||
write = true
|
||||
seqno = s.nextBatch()
|
||||
}
|
||||
|
||||
s.persist = true
|
||||
s.mx.Unlock()
|
||||
|
||||
if write {
|
||||
// all writes sync once perist is true
|
||||
return s.write(seqno)
|
||||
}
|
||||
|
||||
// wait for any pending writes and sync
|
||||
s.mx.Lock()
|
||||
for s.writers > 0 {
|
||||
s.cond.Wait()
|
||||
}
|
||||
s.mx.Unlock()
|
||||
|
||||
return s.db.Sync()
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) EndCriticalSection() {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
s.persist = false
|
||||
return os.RemoveAll(e.path)
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Mark(c cid.Cid) error {
|
||||
s.mx.Lock()
|
||||
|
||||
if s.pend == nil {
|
||||
s.mx.Unlock()
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
write, seqno := s.put(string(c.Hash()))
|
||||
s.mx.Unlock()
|
||||
s.pend[string(c.Hash())] = struct{}{}
|
||||
|
||||
if write {
|
||||
return s.write(seqno)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
|
||||
s.mx.Lock()
|
||||
if s.pend == nil {
|
||||
if len(s.pend) < badgerMarkSetBatchSize {
|
||||
s.mx.Unlock()
|
||||
return errMarkSetClosed
|
||||
return nil
|
||||
}
|
||||
|
||||
write, seqno := s.putMany(batch)
|
||||
s.mx.Unlock()
|
||||
|
||||
if write {
|
||||
return s.write(seqno)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
|
||||
key := c.Hash()
|
||||
pendKey := string(key)
|
||||
|
||||
has, err := s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
}
|
||||
|
||||
return s.tryDB(key)
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Visit(c cid.Cid) (bool, error) {
|
||||
key := c.Hash()
|
||||
pendKey := string(key)
|
||||
|
||||
s.mx.RLock()
|
||||
|
||||
has, err := s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
s.mx.RUnlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
has, err = s.tryDB(key)
|
||||
if has || err != nil {
|
||||
s.mx.RUnlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
// we need to upgrade the lock to exclusive in order to write; take the version count to see
|
||||
// if there was another write while we were upgrading
|
||||
version := s.version
|
||||
s.mx.RUnlock()
|
||||
|
||||
s.mx.Lock()
|
||||
// we have to do the check dance again
|
||||
has, err = s.tryPending(pendKey)
|
||||
if has || err != nil {
|
||||
s.mx.Unlock()
|
||||
return false, err
|
||||
}
|
||||
|
||||
if version != s.version {
|
||||
// something was written to the db, we need to check it
|
||||
has, err = s.tryDB(key)
|
||||
if has || err != nil {
|
||||
s.mx.Unlock()
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
write, seqno := s.put(pendKey)
|
||||
s.mx.Unlock()
|
||||
|
||||
if write {
|
||||
err = s.write(seqno)
|
||||
}
|
||||
|
||||
return true, err
|
||||
}
|
||||
|
||||
// reader holds the (r)lock
|
||||
func (s *BadgerMarkSet) tryPending(key string) (has bool, err error) {
|
||||
if s.pend == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
if _, ok := s.pend[key]; ok {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
for _, wr := range s.writing {
|
||||
if _, ok := wr[key]; ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
|
||||
err = s.db.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(key)
|
||||
return err
|
||||
})
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
return true, nil
|
||||
|
||||
case badger.ErrKeyNotFound:
|
||||
return false, nil
|
||||
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// writer holds the exclusive lock
|
||||
func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
|
||||
s.pend[key] = struct{}{}
|
||||
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
seqno = s.nextBatch()
|
||||
return true, seqno
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
|
||||
for _, c := range batch {
|
||||
key := string(c.Hash())
|
||||
s.pend[key] = struct{}{}
|
||||
}
|
||||
|
||||
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
seqno = s.nextBatch()
|
||||
return true, seqno
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) nextBatch() int {
|
||||
pend := s.pend
|
||||
seqno := s.seqno
|
||||
s.seqno++
|
||||
s.writing[seqno] = s.pend
|
||||
s.writing[seqno] = pend
|
||||
s.pend = make(map[string]struct{})
|
||||
return seqno
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) write(seqno int) (err error) {
|
||||
s.mx.Lock()
|
||||
if s.pend == nil {
|
||||
s.mx.Unlock()
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
pend := s.writing[seqno]
|
||||
s.writers++
|
||||
s.mx.Unlock()
|
||||
|
||||
@@ -320,11 +123,7 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if err == nil {
|
||||
delete(s.writing, seqno)
|
||||
s.version++
|
||||
}
|
||||
|
||||
delete(s.writing, seqno)
|
||||
s.writers--
|
||||
if s.writers == 0 {
|
||||
s.cond.Broadcast()
|
||||
@@ -337,25 +136,56 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
|
||||
defer batch.Cancel()
|
||||
|
||||
for k := range pend {
|
||||
if err = batch.Set([]byte(k), empty); err != nil {
|
||||
return xerrors.Errorf("error setting batch: %w", err)
|
||||
if err := batch.Set([]byte(k), empty); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = batch.Flush()
|
||||
err := batch.Flush()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error flushing batch to badger markset: %w", err)
|
||||
}
|
||||
|
||||
s.mx.RLock()
|
||||
persist := s.persist
|
||||
s.mx.RUnlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
if persist && !system.BadgerFsyncDisable { // WARNING: disabling sync makes recovery from crash during critical section unsound
|
||||
return s.db.Sync()
|
||||
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
|
||||
if s.pend == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
return nil
|
||||
key := c.Hash()
|
||||
pendKey := string(key)
|
||||
_, ok := s.pend[pendKey]
|
||||
if ok {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
for _, wr := range s.writing {
|
||||
_, ok := wr[pendKey]
|
||||
if ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(key)
|
||||
return err
|
||||
})
|
||||
|
||||
switch err {
|
||||
case nil:
|
||||
return true, nil
|
||||
|
||||
case badger.ErrKeyNotFound:
|
||||
return false, nil
|
||||
|
||||
default:
|
||||
return false, xerrors.Errorf("error checking badger markset: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) Close() error {
|
||||
@@ -374,65 +204,12 @@ func (s *BadgerMarkSet) Close() error {
|
||||
db := s.db
|
||||
s.db = nil
|
||||
|
||||
return closeBadgerDB(db, s.path, s.persist)
|
||||
}
|
||||
|
||||
func openBadgerDB(path string, recover bool) (*badger.DB, error) {
|
||||
// if it is not a recovery, clean up first
|
||||
if !recover {
|
||||
err := os.RemoveAll(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
|
||||
}
|
||||
|
||||
err = os.MkdirAll(path, 0755) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating markset directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
opts := badger.DefaultOptions(path)
|
||||
// we manually sync when we are in critical section
|
||||
opts.SyncWrites = false
|
||||
// no need to do that
|
||||
opts.CompactL0OnClose = false
|
||||
// we store hashes, not much to gain by compression
|
||||
opts.Compression = options.None
|
||||
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
|
||||
// between the system blockstore and the markset.
|
||||
// It was observed that using the default memory mapped option resulted in
|
||||
// significant interference and unacceptably high block validation times once the markset
|
||||
// exceeded 1GB in size.
|
||||
opts.TableLoadingMode = options.FileIO
|
||||
opts.ValueLogLoadingMode = options.FileIO
|
||||
// We increase the number of L0 tables before compaction to make it unlikely to
|
||||
// be necessary.
|
||||
opts.NumLevelZeroTables = 20 // default is 5
|
||||
opts.NumLevelZeroTablesStall = 30 // default is 10
|
||||
// increase the number of compactors from default 2 so that if we ever have to
|
||||
// compact, it is fast
|
||||
if runtime.NumCPU()/2 > opts.NumCompactors {
|
||||
opts.NumCompactors = runtime.NumCPU() / 2
|
||||
}
|
||||
opts.Logger = &badgerLogger{
|
||||
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
|
||||
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
|
||||
}
|
||||
|
||||
return badger.Open(opts)
|
||||
}
|
||||
|
||||
func closeBadgerDB(db *badger.DB, path string, persist bool) error {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error closing badger markset: %w", err)
|
||||
}
|
||||
|
||||
if persist {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = os.RemoveAll(path)
|
||||
err = os.RemoveAll(s.path)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error deleting badger markset: %w", err)
|
||||
}
|
||||
@@ -440,6 +217,8 @@ func closeBadgerDB(db *badger.DB, path string, persist bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BadgerMarkSet) SetConcurrent() {}
|
||||
|
||||
// badger logging through go-log
|
||||
type badgerLogger struct {
|
||||
*zap.SugaredLogger
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
bbloom "github.com/ipfs/bbloom"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
const (
|
||||
BloomFilterMinSize = 10_000_000
|
||||
BloomFilterProbability = 0.01
|
||||
)
|
||||
|
||||
type BloomMarkSetEnv struct{}
|
||||
|
||||
var _ MarkSetEnv = (*BloomMarkSetEnv)(nil)
|
||||
|
||||
type BloomMarkSet struct {
|
||||
salt []byte
|
||||
mx sync.RWMutex
|
||||
bf *bbloom.Bloom
|
||||
ts bool
|
||||
}
|
||||
|
||||
var _ MarkSet = (*BloomMarkSet)(nil)
|
||||
|
||||
func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error) {
|
||||
return &BloomMarkSetEnv{}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
size := int64(BloomFilterMinSize)
|
||||
for size < sizeHint {
|
||||
size += BloomFilterMinSize
|
||||
}
|
||||
|
||||
salt := make([]byte, 4)
|
||||
_, err := rand.Read(salt)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error reading salt: %w", err)
|
||||
}
|
||||
|
||||
bf, err := bbloom.New(float64(size), BloomFilterProbability)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating bloom filter: %w", err)
|
||||
}
|
||||
|
||||
return &BloomMarkSet{salt: salt, bf: bf}, nil
|
||||
}
|
||||
|
||||
func (e *BloomMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) saltedKey(cid cid.Cid) []byte {
|
||||
hash := cid.Hash()
|
||||
key := make([]byte, len(s.salt)+len(hash))
|
||||
n := copy(key, s.salt)
|
||||
copy(key[n:], hash)
|
||||
rehash := sha256.Sum256(key)
|
||||
return rehash[:]
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Mark(cid cid.Cid) error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
|
||||
if s.bf == nil {
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
s.bf.Add(s.saltedKey(cid))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
if s.ts {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
}
|
||||
|
||||
if s.bf == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
return s.bf.Has(s.saltedKey(cid)), nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) Close() error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
s.bf = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BloomMarkSet) SetConcurrent() {
|
||||
s.ts = true
|
||||
}
|
||||
@@ -1,19 +1,12 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type MapMarkSetEnv struct {
|
||||
path string
|
||||
}
|
||||
type MapMarkSetEnv struct{}
|
||||
|
||||
var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
|
||||
|
||||
@@ -21,194 +14,44 @@ type MapMarkSet struct {
|
||||
mx sync.RWMutex
|
||||
set map[string]struct{}
|
||||
|
||||
persist bool
|
||||
file *os.File
|
||||
buf *bufio.Writer
|
||||
|
||||
path string
|
||||
ts bool
|
||||
}
|
||||
|
||||
var _ MarkSet = (*MapMarkSet)(nil)
|
||||
|
||||
func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
|
||||
msPath := filepath.Join(path, "markset.map")
|
||||
err := os.MkdirAll(msPath, 0755) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error creating markset directory: %w", err)
|
||||
}
|
||||
|
||||
return &MapMarkSetEnv{path: msPath}, nil
|
||||
func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
|
||||
return &MapMarkSetEnv{}, nil
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
|
||||
path := filepath.Join(e.path, name)
|
||||
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
|
||||
return &MapMarkSet{
|
||||
set: make(map[string]struct{}, sizeHint),
|
||||
path: path,
|
||||
set: make(map[string]struct{}, sizeHint),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
|
||||
path := filepath.Join(e.path, name)
|
||||
s := &MapMarkSet{
|
||||
set: make(map[string]struct{}),
|
||||
path: path,
|
||||
}
|
||||
|
||||
in, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error opening markset file for read: %w", err)
|
||||
}
|
||||
defer in.Close() //nolint:errcheck
|
||||
|
||||
// wrap a buffered reader to make this faster
|
||||
buf := bufio.NewReader(in)
|
||||
for {
|
||||
var sz byte
|
||||
if sz, err = buf.ReadByte(); err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
key := make([]byte, int(sz))
|
||||
if _, err = io.ReadFull(buf, key); err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
s.set[string(key)] = struct{}{}
|
||||
}
|
||||
|
||||
if err != io.EOF {
|
||||
return nil, xerrors.Errorf("error reading markset file: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("error opening markset file for write: %w", err)
|
||||
}
|
||||
|
||||
s.persist = true
|
||||
s.file = file
|
||||
s.buf = bufio.NewWriter(file)
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (e *MapMarkSetEnv) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) BeginCriticalSection() error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
func (s *MapMarkSet) Mark(cid cid.Cid) error {
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
|
||||
if s.set == nil {
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
if s.persist {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error opening markset file: %w", err)
|
||||
}
|
||||
|
||||
// wrap a buffered writer to make this faster
|
||||
s.buf = bufio.NewWriter(file)
|
||||
for key := range s.set {
|
||||
if err := s.writeKey([]byte(key), false); err != nil {
|
||||
_ = file.Close()
|
||||
s.buf = nil
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
_ = file.Close()
|
||||
s.buf = nil
|
||||
return xerrors.Errorf("error flushing markset file buffer: %w", err)
|
||||
}
|
||||
|
||||
s.file = file
|
||||
s.persist = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) EndCriticalSection() {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if !s.persist {
|
||||
return
|
||||
}
|
||||
|
||||
_ = s.file.Close()
|
||||
_ = os.Remove(s.path)
|
||||
s.file = nil
|
||||
s.buf = nil
|
||||
s.persist = false
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Mark(c cid.Cid) error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if s.set == nil {
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
hash := c.Hash()
|
||||
s.set[string(hash)] = struct{}{}
|
||||
|
||||
if s.persist {
|
||||
if err := s.writeKey(hash, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.file.Sync(); err != nil {
|
||||
return xerrors.Errorf("error syncing markset: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if s.set == nil {
|
||||
return errMarkSetClosed
|
||||
}
|
||||
|
||||
for _, c := range batch {
|
||||
hash := c.Hash()
|
||||
s.set[string(hash)] = struct{}{}
|
||||
|
||||
if s.persist {
|
||||
if err := s.writeKey(hash, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.persist {
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
|
||||
}
|
||||
|
||||
if err := s.file.Sync(); err != nil {
|
||||
return xerrors.Errorf("error syncing markset: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.set[string(cid.Hash())] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
if s.ts {
|
||||
s.mx.RLock()
|
||||
defer s.mx.RUnlock()
|
||||
}
|
||||
|
||||
if s.set == nil {
|
||||
return false, errMarkSetClosed
|
||||
@@ -218,71 +61,15 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if s.set == nil {
|
||||
return false, errMarkSetClosed
|
||||
}
|
||||
|
||||
hash := c.Hash()
|
||||
key := string(hash)
|
||||
if _, ok := s.set[key]; ok {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
s.set[key] = struct{}{}
|
||||
|
||||
if s.persist {
|
||||
if err := s.writeKey(hash, true); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.file.Sync(); err != nil {
|
||||
return false, xerrors.Errorf("error syncing markset: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) Close() error {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
|
||||
if s.set == nil {
|
||||
return nil
|
||||
if s.ts {
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
}
|
||||
|
||||
s.set = nil
|
||||
|
||||
if s.file != nil {
|
||||
if err := s.file.Close(); err != nil {
|
||||
log.Warnf("error closing markset file: %s", err)
|
||||
}
|
||||
|
||||
if !s.persist {
|
||||
if err := os.Remove(s.path); err != nil {
|
||||
log.Warnf("error removing markset file: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
|
||||
if err := s.buf.WriteByte(byte(len(k))); err != nil {
|
||||
return xerrors.Errorf("error writing markset key length to disk: %w", err)
|
||||
}
|
||||
if _, err := s.buf.Write(k); err != nil {
|
||||
return xerrors.Errorf("error writing markset key to disk: %w", err)
|
||||
}
|
||||
if flush {
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
func (s *MapMarkSet) SetConcurrent() {
|
||||
s.ts = true
|
||||
}
|
||||
|
||||
@@ -1,40 +1,37 @@
|
||||
// stm: #unit
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
func TestMapMarkSet(t *testing.T) {
|
||||
//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
|
||||
//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
|
||||
testMarkSet(t, "map")
|
||||
testMarkSetRecovery(t, "map")
|
||||
testMarkSetMarkMany(t, "map")
|
||||
testMarkSetVisitor(t, "map")
|
||||
testMarkSetVisitorRecovery(t, "map")
|
||||
}
|
||||
|
||||
func TestBloomMarkSet(t *testing.T) {
|
||||
testMarkSet(t, "bloom")
|
||||
}
|
||||
|
||||
func TestBadgerMarkSet(t *testing.T) {
|
||||
//stm: @SPLITSTORE_MARKSET_CREATE_001, @SPLITSTORE_MARKSET_HAS_001, @@SPLITSTORE_MARKSET_MARK_001
|
||||
//stm: @SPLITSTORE_MARKSET_CLOSE_001, @SPLITSTORE_MARKSET_CREATE_VISITOR_001
|
||||
bs := badgerMarkSetBatchSize
|
||||
badgerMarkSetBatchSize = 1
|
||||
t.Cleanup(func() {
|
||||
badgerMarkSetBatchSize = bs
|
||||
})
|
||||
testMarkSet(t, "badger")
|
||||
testMarkSetRecovery(t, "badger")
|
||||
testMarkSetMarkMany(t, "badger")
|
||||
testMarkSetVisitor(t, "badger")
|
||||
testMarkSetVisitorRecovery(t, "badger")
|
||||
}
|
||||
|
||||
func testMarkSet(t *testing.T, lsType string) {
|
||||
path := t.TempDir()
|
||||
t.Helper()
|
||||
|
||||
path, err := ioutil.TempDir("", "sweep-test.*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
@@ -42,13 +39,12 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
// stm: @SPLITSTORE_MARKSET_CREATE_001
|
||||
hotSet, err := env.New("hot", 0)
|
||||
hotSet, err := env.Create("hot", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
coldSet, err := env.New("cold", 0)
|
||||
coldSet, err := env.Create("cold", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -62,9 +58,7 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
// stm: @SPLITSTORE_MARKSET_HAS_001
|
||||
mustHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -76,7 +70,6 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
}
|
||||
|
||||
mustNotHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -92,7 +85,6 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
// stm: @SPLITSTORE_MARKSET_MARK_001
|
||||
hotSet.Mark(k1) //nolint
|
||||
hotSet.Mark(k2) //nolint
|
||||
coldSet.Mark(k3) //nolint
|
||||
@@ -119,12 +111,12 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hotSet, err = env.New("hot", 0)
|
||||
hotSet, err = env.Create("hot", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
coldSet, err = env.New("cold", 0)
|
||||
coldSet, err = env.Create("cold", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -143,7 +135,6 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
mustNotHave(coldSet, k3)
|
||||
mustNotHave(coldSet, k4)
|
||||
|
||||
//stm: @SPLITSTORE_MARKSET_CLOSE_001
|
||||
err = hotSet.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -154,364 +145,3 @@ func testMarkSet(t *testing.T, lsType string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func testMarkSetVisitor(t *testing.T, lsType string) {
|
||||
path := t.TempDir()
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
//stm: @SPLITSTORE_MARKSET_CREATE_VISITOR_001
|
||||
visitor, err := env.New("test", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer visitor.Close() //nolint:errcheck
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
t.Fatal("object should be visited")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if visit {
|
||||
t.Fatal("unexpected visit")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
mustVisit(visitor, k1)
|
||||
mustVisit(visitor, k2)
|
||||
mustVisit(visitor, k3)
|
||||
mustVisit(visitor, k4)
|
||||
|
||||
mustNotVisit(visitor, k1)
|
||||
mustNotVisit(visitor, k2)
|
||||
mustNotVisit(visitor, k3)
|
||||
mustNotVisit(visitor, k4)
|
||||
}
|
||||
|
||||
func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
|
||||
path := t.TempDir()
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
visitor, err := env.New("test", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer visitor.Close() //nolint:errcheck
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !visit {
|
||||
t.Fatal("object should be visited")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
|
||||
visit, err := v.Visit(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if visit {
|
||||
t.Fatal("unexpected visit")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
mustVisit(visitor, k1)
|
||||
mustVisit(visitor, k2)
|
||||
|
||||
if err := visitor.BeginCriticalSection(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustVisit(visitor, k3)
|
||||
mustVisit(visitor, k4)
|
||||
|
||||
mustNotVisit(visitor, k1)
|
||||
mustNotVisit(visitor, k2)
|
||||
mustNotVisit(visitor, k3)
|
||||
mustNotVisit(visitor, k4)
|
||||
|
||||
if err := visitor.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
visitor, err = env.Recover("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustNotVisit(visitor, k1)
|
||||
mustNotVisit(visitor, k2)
|
||||
mustNotVisit(visitor, k3)
|
||||
mustNotVisit(visitor, k4)
|
||||
|
||||
visitor.EndCriticalSection()
|
||||
|
||||
if err := visitor.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = env.Recover("test")
|
||||
if err == nil {
|
||||
t.Fatal("expected recovery to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func testMarkSetRecovery(t *testing.T, lsType string) {
|
||||
path := t.TempDir()
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
markSet, err := env.New("test", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("mark not found")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("unexpected mark")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
if err := markSet.Mark(k1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := markSet.Mark(k2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustNotHave(markSet, k3)
|
||||
mustNotHave(markSet, k4)
|
||||
|
||||
if err := markSet.BeginCriticalSection(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := markSet.Mark(k3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := markSet.Mark(k4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustHave(markSet, k3)
|
||||
mustHave(markSet, k4)
|
||||
|
||||
if err := markSet.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
markSet, err = env.Recover("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustHave(markSet, k3)
|
||||
mustHave(markSet, k4)
|
||||
|
||||
markSet.EndCriticalSection()
|
||||
|
||||
if err := markSet.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = env.Recover("test")
|
||||
if err == nil {
|
||||
t.Fatal("expected recovery to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func testMarkSetMarkMany(t *testing.T, lsType string) {
|
||||
path := t.TempDir()
|
||||
|
||||
env, err := OpenMarkSetEnv(path, lsType)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer env.Close() //nolint:errcheck
|
||||
|
||||
markSet, err := env.New("test", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
makeCid := func(key string) cid.Cid {
|
||||
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
mustHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatal("mark not found")
|
||||
}
|
||||
}
|
||||
|
||||
mustNotHave := func(s MarkSet, cid cid.Cid) {
|
||||
t.Helper()
|
||||
has, err := s.Has(cid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("unexpected mark")
|
||||
}
|
||||
}
|
||||
|
||||
k1 := makeCid("a")
|
||||
k2 := makeCid("b")
|
||||
k3 := makeCid("c")
|
||||
k4 := makeCid("d")
|
||||
|
||||
if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustNotHave(markSet, k3)
|
||||
mustNotHave(markSet, k4)
|
||||
|
||||
if err := markSet.BeginCriticalSection(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustHave(markSet, k3)
|
||||
mustHave(markSet, k4)
|
||||
|
||||
if err := markSet.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
markSet, err = env.Recover("test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mustHave(markSet, k1)
|
||||
mustHave(markSet, k2)
|
||||
mustHave(markSet, k3)
|
||||
mustHave(markSet, k4)
|
||||
|
||||
markSet.EndCriticalSection()
|
||||
|
||||
if err := markSet.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = env.Recover("test")
|
||||
if err == nil {
|
||||
t.Fatal("expected recovery to fail")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,22 +8,20 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
dstore "github.com/ipfs/go-datastore"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"go.opencensus.io/stats"
|
||||
"go.uber.org/multierr"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
dstore "github.com/ipfs/go-datastore"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
|
||||
"go.opencensus.io/stats"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -43,33 +41,12 @@ var (
|
||||
// compactionIndexKey stores the compaction index (serial number)
|
||||
compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
|
||||
|
||||
// stores the prune index (serial number)
|
||||
pruneIndexKey = dstore.NewKey("/splitstore/pruneIndex")
|
||||
|
||||
// stores the base epoch of last prune in the metadata store
|
||||
pruneEpochKey = dstore.NewKey("/splitstore/pruneEpoch")
|
||||
|
||||
log = logging.Logger("splitstore")
|
||||
|
||||
errClosing = errors.New("splitstore is closing")
|
||||
|
||||
// set this to true if you are debugging the splitstore to enable debug logging
|
||||
enableDebugLog = false
|
||||
// set this to true if you want to track origin stack traces in the write log
|
||||
enableDebugLogWriteTraces = false
|
||||
|
||||
// upgradeBoundary is the boundary before and after an upgrade where we suppress compaction
|
||||
upgradeBoundary = build.Finality
|
||||
)
|
||||
|
||||
type CompactType int
|
||||
|
||||
const (
|
||||
none CompactType = iota
|
||||
warmup
|
||||
hot
|
||||
cold
|
||||
check
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -98,10 +75,6 @@ type Config struct {
|
||||
// and directly purges cold blocks.
|
||||
DiscardColdBlocks bool
|
||||
|
||||
// UniversalColdBlocks indicates whether all blocks being garbage collected and purged
|
||||
// from the hotstore should be written to the cold store
|
||||
UniversalColdBlocks bool
|
||||
|
||||
// HotstoreMessageRetention indicates the hotstore retention policy for messages.
|
||||
// It has the following semantics:
|
||||
// - a value of 0 will only retain messages within the compaction boundary (4 finalities)
|
||||
@@ -115,23 +88,6 @@ type Config struct {
|
||||
// A positive value is the number of compactions before a full GC is performed;
|
||||
// a value of 1 will perform full GC in every compaction.
|
||||
HotStoreFullGCFrequency uint64
|
||||
|
||||
// HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take.
|
||||
// This is not a hard limit, it is possible for the hotstore to exceed the target
|
||||
// for example if state grows massively between compactions. The splitstore
|
||||
// will make a best effort to avoid overflowing the target and in practice should
|
||||
// never overflow. This field is used when doing GC at the end of a compaction to
|
||||
// adaptively choose moving GC
|
||||
HotstoreMaxSpaceTarget uint64
|
||||
|
||||
// Moving GC will be triggered when total moving size exceeds
|
||||
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
|
||||
HotstoreMaxSpaceThreshold uint64
|
||||
|
||||
// Safety buffer to prevent moving GC from overflowing disk.
|
||||
// Moving GC will not occur when total moving size exceeds
|
||||
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
|
||||
HotstoreMaxSpaceSafetyBuffer uint64
|
||||
}
|
||||
|
||||
// ChainAccessor allows the Splitstore to access the chain. It will most likely
|
||||
@@ -142,12 +98,6 @@ type ChainAccessor interface {
|
||||
SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
|
||||
}
|
||||
|
||||
// upgradeRange is a precomputed epoch range during which we shouldn't compact so as to not
|
||||
// interfere with an upgrade
|
||||
type upgradeRange struct {
|
||||
start, end abi.ChainEpoch
|
||||
}
|
||||
|
||||
// hotstore is the interface that must be satisfied by the hot blockstore; it is an extension
|
||||
// of the Blockstore interface with the traits we need for compaction.
|
||||
type hotstore interface {
|
||||
@@ -156,41 +106,33 @@ type hotstore interface {
|
||||
}
|
||||
|
||||
type SplitStore struct {
|
||||
compacting int32 // flag for when compaction is in progress
|
||||
compactType CompactType // compaction type, protected by compacting atomic, only meaningful when compacting == 1
|
||||
closing int32 // the splitstore is closing
|
||||
compacting int32 // compaction/prune/warmup in progress
|
||||
closing int32 // the splitstore is closing
|
||||
|
||||
cfg *Config
|
||||
path string
|
||||
|
||||
mx sync.Mutex
|
||||
warmupEpoch atomic.Int64
|
||||
warmupEpoch abi.ChainEpoch // protected by mx
|
||||
baseEpoch abi.ChainEpoch // protected by compaction lock
|
||||
pruneEpoch abi.ChainEpoch // protected by compaction lock
|
||||
|
||||
headChangeMx sync.Mutex
|
||||
|
||||
coldPurgeSize int
|
||||
|
||||
chain ChainAccessor
|
||||
ds dstore.Datastore
|
||||
cold bstore.Blockstore
|
||||
hot hotstore
|
||||
|
||||
upgrades []upgradeRange
|
||||
|
||||
markSetEnv MarkSetEnv
|
||||
markSetSize int64
|
||||
|
||||
compactionIndex int64
|
||||
pruneIndex int64
|
||||
|
||||
ctx context.Context
|
||||
cancel func()
|
||||
|
||||
outOfSync int32 // for fast checking
|
||||
chainSyncMx sync.Mutex
|
||||
chainSyncCond sync.Cond
|
||||
chainSyncFinished bool // protected by chainSyncMx
|
||||
|
||||
debug *debugLog
|
||||
|
||||
// transactional protection for concurrent read/writes during compaction
|
||||
@@ -200,34 +142,13 @@ type SplitStore struct {
|
||||
txnViews int
|
||||
txnViewsWaiting bool
|
||||
txnActive bool
|
||||
txnProtect MarkSet
|
||||
txnRefsMx sync.Mutex
|
||||
txnRefs map[cid.Cid]struct{}
|
||||
txnMissing map[cid.Cid]struct{}
|
||||
txnMarkSet MarkSet
|
||||
txnSyncMx sync.Mutex
|
||||
txnSyncCond sync.Cond
|
||||
txnSync bool
|
||||
|
||||
// background cold object reification
|
||||
reifyWorkers sync.WaitGroup
|
||||
reifyMx sync.Mutex
|
||||
reifyCond sync.Cond
|
||||
reifyPend map[cid.Cid]struct{}
|
||||
reifyInProgress map[cid.Cid]struct{}
|
||||
|
||||
// registered protectors
|
||||
protectors []func(func(cid.Cid) error) error
|
||||
|
||||
// dag sizes measured during latest compaction
|
||||
// logged and used for GC strategy
|
||||
|
||||
// protected by compaction lock
|
||||
szWalk int64
|
||||
szProtectedTxns int64
|
||||
szKeys int64 // approximate, not counting keys protected when entering critical section
|
||||
|
||||
// protected by txnLk
|
||||
szMarkedLiveRefs int64
|
||||
}
|
||||
|
||||
var _ bstore.Blockstore = (*SplitStore)(nil)
|
||||
@@ -261,17 +182,13 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
|
||||
cold: cold,
|
||||
hot: hots,
|
||||
markSetEnv: markSetEnv,
|
||||
|
||||
coldPurgeSize: defaultColdPurgeSize,
|
||||
}
|
||||
|
||||
ss.txnViewsCond.L = &ss.txnViewsMx
|
||||
ss.txnSyncCond.L = &ss.txnSyncMx
|
||||
ss.chainSyncCond.L = &ss.chainSyncMx
|
||||
ss.ctx, ss.cancel = context.WithCancel(context.Background())
|
||||
|
||||
ss.reifyCond.L = &ss.reifyMx
|
||||
ss.reifyPend = make(map[cid.Cid]struct{})
|
||||
ss.reifyInProgress = make(map[cid.Cid]struct{})
|
||||
|
||||
if enableDebugLog {
|
||||
ss.debug, err = openDebugLog(path)
|
||||
if err != nil {
|
||||
@@ -279,36 +196,21 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
|
||||
}
|
||||
}
|
||||
|
||||
if ss.checkpointExists() {
|
||||
log.Info("found compaction checkpoint; resuming compaction")
|
||||
if err := ss.completeCompaction(); err != nil {
|
||||
markSetEnv.Close() //nolint:errcheck
|
||||
return nil, xerrors.Errorf("error resuming compaction: %w", err)
|
||||
}
|
||||
}
|
||||
if ss.pruneCheckpointExists() {
|
||||
log.Info("found prune checkpoint; resuming prune")
|
||||
if err := ss.completePrune(); err != nil {
|
||||
markSetEnv.Close() //nolint:errcheck
|
||||
return nil, xerrors.Errorf("error resuming prune: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
// Blockstore interface
|
||||
func (s *SplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
|
||||
func (s *SplitStore) DeleteBlock(_ cid.Cid) error {
|
||||
// afaict we don't seem to be using this method, so it's not implemented
|
||||
return errors.New("DeleteBlock not implemented on SplitStore; don't do this Luke!") //nolint
|
||||
}
|
||||
|
||||
func (s *SplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
|
||||
func (s *SplitStore) DeleteMany(_ []cid.Cid) error {
|
||||
// afaict we don't seem to be using this method, so it's not implemented
|
||||
return errors.New("DeleteMany not implemented on SplitStore; don't do this Luke!") //nolint
|
||||
}
|
||||
|
||||
func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
func (s *SplitStore) Has(cid cid.Cid) (bool, error) {
|
||||
if isIdentiyCid(cid) {
|
||||
return true, nil
|
||||
}
|
||||
@@ -316,27 +218,7 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
// critical section
|
||||
if s.txnMarkSet != nil {
|
||||
has, err := s.txnMarkSet.Has(cid)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if has {
|
||||
return s.has(cid)
|
||||
}
|
||||
switch s.compactType {
|
||||
case hot:
|
||||
return s.cold.Has(ctx, cid)
|
||||
case cold:
|
||||
return s.hot.Has(ctx, cid)
|
||||
default:
|
||||
return false, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
|
||||
}
|
||||
}
|
||||
|
||||
has, err := s.hot.Has(ctx, cid)
|
||||
has, err := s.hot.Has(cid)
|
||||
|
||||
if err != nil {
|
||||
return has, err
|
||||
@@ -347,19 +229,10 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
has, err = s.cold.Has(ctx, cid)
|
||||
if has {
|
||||
s.trackTxnRef(cid)
|
||||
if bstore.IsHotView(ctx) {
|
||||
s.reifyColdObject(cid)
|
||||
}
|
||||
}
|
||||
|
||||
return has, err
|
||||
|
||||
return s.cold.Has(cid)
|
||||
}
|
||||
|
||||
func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
if isIdentiyCid(cid) {
|
||||
data, err := decodeIdentityCid(cid)
|
||||
if err != nil {
|
||||
@@ -372,46 +245,22 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
// critical section
|
||||
if s.txnMarkSet != nil {
|
||||
has, err := s.txnMarkSet.Has(cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blk, err := s.hot.Get(cid)
|
||||
|
||||
if has {
|
||||
return s.get(cid)
|
||||
}
|
||||
switch s.compactType {
|
||||
case hot:
|
||||
return s.cold.Get(ctx, cid)
|
||||
case cold:
|
||||
return s.hot.Get(ctx, cid)
|
||||
default:
|
||||
return nil, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
|
||||
}
|
||||
}
|
||||
|
||||
blk, err := s.hot.Get(ctx, cid)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
switch err {
|
||||
case nil:
|
||||
s.trackTxnRef(cid)
|
||||
return blk, nil
|
||||
|
||||
case ipld.IsNotFound(err):
|
||||
case bstore.ErrNotFound:
|
||||
if s.isWarm() {
|
||||
s.debug.LogReadMiss(cid)
|
||||
}
|
||||
|
||||
blk, err = s.cold.Get(ctx, cid)
|
||||
blk, err = s.cold.Get(cid)
|
||||
if err == nil {
|
||||
s.trackTxnRef(cid)
|
||||
if bstore.IsHotView(ctx) {
|
||||
s.reifyColdObject(cid)
|
||||
}
|
||||
|
||||
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
|
||||
|
||||
}
|
||||
return blk, err
|
||||
|
||||
@@ -420,7 +269,7 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
func (s *SplitStore) GetSize(cid cid.Cid) (int, error) {
|
||||
if isIdentiyCid(cid) {
|
||||
data, err := decodeIdentityCid(cid)
|
||||
if err != nil {
|
||||
@@ -433,45 +282,20 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
// critical section
|
||||
if s.txnMarkSet != nil {
|
||||
has, err := s.txnMarkSet.Has(cid)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
size, err := s.hot.GetSize(cid)
|
||||
|
||||
if has {
|
||||
return s.getSize(cid)
|
||||
}
|
||||
switch s.compactType {
|
||||
case hot:
|
||||
return s.cold.GetSize(ctx, cid)
|
||||
case cold:
|
||||
return s.hot.GetSize(ctx, cid)
|
||||
default:
|
||||
return 0, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
|
||||
}
|
||||
}
|
||||
|
||||
size, err := s.hot.GetSize(ctx, cid)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
switch err {
|
||||
case nil:
|
||||
s.trackTxnRef(cid)
|
||||
return size, nil
|
||||
|
||||
case ipld.IsNotFound(err):
|
||||
case bstore.ErrNotFound:
|
||||
if s.isWarm() {
|
||||
s.debug.LogReadMiss(cid)
|
||||
}
|
||||
|
||||
size, err = s.cold.GetSize(ctx, cid)
|
||||
size, err = s.cold.GetSize(cid)
|
||||
if err == nil {
|
||||
s.trackTxnRef(cid)
|
||||
if bstore.IsHotView(ctx) {
|
||||
s.reifyColdObject(cid)
|
||||
}
|
||||
|
||||
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
|
||||
}
|
||||
return size, err
|
||||
@@ -481,24 +305,7 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) Flush(ctx context.Context) error {
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
if err := s.cold.Flush(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.hot.Flush(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.ds.Sync(ctx, dstore.Key{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
func (s *SplitStore) Put(blk blocks.Block) error {
|
||||
if isIdentiyCid(blk.Cid()) {
|
||||
return nil
|
||||
}
|
||||
@@ -506,24 +313,18 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
err := s.hot.Put(ctx, blk)
|
||||
err := s.hot.Put(blk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.debug.LogWrite(blk)
|
||||
|
||||
// critical section
|
||||
if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
|
||||
s.markLiveRefs([]cid.Cid{blk.Cid()})
|
||||
return nil
|
||||
}
|
||||
s.trackTxnRef(blk.Cid())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
func (s *SplitStore) PutMany(blks []blocks.Block) error {
|
||||
// filter identites
|
||||
idcids := 0
|
||||
for _, blk := range blks {
|
||||
@@ -557,20 +358,14 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
err := s.hot.PutMany(ctx, blks)
|
||||
err := s.hot.PutMany(blks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.debug.LogWriteMany(blks)
|
||||
|
||||
// critical section
|
||||
if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
|
||||
s.markLiveRefs(batch)
|
||||
return nil
|
||||
}
|
||||
s.trackTxnRefMany(batch)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -619,7 +414,7 @@ func (s *SplitStore) HashOnRead(enabled bool) {
|
||||
s.cold.HashOnRead(enabled)
|
||||
}
|
||||
|
||||
func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) error) error {
|
||||
func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error {
|
||||
if isIdentiyCid(cid) {
|
||||
data, err := decodeIdentityCid(cid)
|
||||
if err != nil {
|
||||
@@ -629,29 +424,6 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
|
||||
return cb(data)
|
||||
}
|
||||
|
||||
// critical section
|
||||
s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
|
||||
if s.txnMarkSet != nil {
|
||||
has, err := s.txnMarkSet.Has(cid)
|
||||
s.txnLk.RUnlock()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if has {
|
||||
return s.view(cid, cb)
|
||||
}
|
||||
switch s.compactType {
|
||||
case hot:
|
||||
return s.cold.View(ctx, cid, cb)
|
||||
case cold:
|
||||
return s.hot.View(ctx, cid, cb)
|
||||
default:
|
||||
return xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
|
||||
}
|
||||
}
|
||||
|
||||
// views are (optimistically) protected two-fold:
|
||||
// - if there is an active transaction, then the reference is protected.
|
||||
// - if there is no active transaction, active views are tracked in a
|
||||
@@ -663,57 +435,41 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
|
||||
s.protectView(cid)
|
||||
defer s.viewDone()
|
||||
|
||||
err := s.hot.View(ctx, cid, cb)
|
||||
if ipld.IsNotFound(err) {
|
||||
err := s.hot.View(cid, cb)
|
||||
switch err {
|
||||
case bstore.ErrNotFound:
|
||||
if s.isWarm() {
|
||||
s.debug.LogReadMiss(cid)
|
||||
}
|
||||
|
||||
err = s.cold.View(ctx, cid, cb)
|
||||
err = s.cold.View(cid, cb)
|
||||
if err == nil {
|
||||
if bstore.IsHotView(ctx) {
|
||||
s.reifyColdObject(cid)
|
||||
}
|
||||
|
||||
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
|
||||
}
|
||||
return err
|
||||
|
||||
default:
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SplitStore) isWarm() bool {
|
||||
return s.warmupEpoch.Load() > 0
|
||||
s.mx.Lock()
|
||||
defer s.mx.Unlock()
|
||||
return s.warmupEpoch > 0
|
||||
}
|
||||
|
||||
// State tracking
|
||||
func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error {
|
||||
func (s *SplitStore) Start(chain ChainAccessor) error {
|
||||
s.chain = chain
|
||||
curTs := chain.GetHeaviestTipSet()
|
||||
|
||||
// precompute the upgrade boundaries
|
||||
s.upgrades = make([]upgradeRange, 0, len(us))
|
||||
for _, upgrade := range us {
|
||||
boundary := upgrade.Height
|
||||
for _, pre := range upgrade.PreMigrations {
|
||||
preMigrationBoundary := upgrade.Height - pre.StartWithin
|
||||
if preMigrationBoundary < boundary {
|
||||
boundary = preMigrationBoundary
|
||||
}
|
||||
}
|
||||
|
||||
upgradeStart := boundary - upgradeBoundary
|
||||
upgradeEnd := upgrade.Height + upgradeBoundary
|
||||
|
||||
s.upgrades = append(s.upgrades, upgradeRange{start: upgradeStart, end: upgradeEnd})
|
||||
}
|
||||
|
||||
// should we warmup
|
||||
warmup := false
|
||||
|
||||
// load base epoch from metadata ds
|
||||
// if none, then use current epoch because it's a fresh start
|
||||
bs, err := s.ds.Get(s.ctx, baseEpochKey)
|
||||
bs, err := s.ds.Get(baseEpochKey)
|
||||
switch err {
|
||||
case nil:
|
||||
s.baseEpoch = bytesToEpoch(bs)
|
||||
@@ -733,28 +489,11 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
|
||||
return xerrors.Errorf("error loading base epoch: %w", err)
|
||||
}
|
||||
|
||||
// load prune epoch from metadata ds
|
||||
bs, err = s.ds.Get(s.ctx, pruneEpochKey)
|
||||
switch err {
|
||||
case nil:
|
||||
s.pruneEpoch = bytesToEpoch(bs)
|
||||
case dstore.ErrNotFound:
|
||||
if curTs == nil {
|
||||
//this can happen in some tests
|
||||
break
|
||||
}
|
||||
if err := s.setPruneEpoch(curTs.Height()); err != nil {
|
||||
return xerrors.Errorf("error saving prune epoch: %w", err)
|
||||
}
|
||||
default:
|
||||
return xerrors.Errorf("error loading prune epoch: %w", err)
|
||||
}
|
||||
|
||||
// load warmup epoch from metadata ds
|
||||
bs, err = s.ds.Get(s.ctx, warmupEpochKey)
|
||||
bs, err = s.ds.Get(warmupEpochKey)
|
||||
switch err {
|
||||
case nil:
|
||||
s.warmupEpoch.Store(bytesToInt64(bs))
|
||||
s.warmupEpoch = bytesToEpoch(bs)
|
||||
|
||||
case dstore.ErrNotFound:
|
||||
warmup = true
|
||||
@@ -764,7 +503,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
|
||||
}
|
||||
|
||||
// load markSetSize from metadata ds to provide a size hint for marksets
|
||||
bs, err = s.ds.Get(s.ctx, markSetSizeKey)
|
||||
bs, err = s.ds.Get(markSetSizeKey)
|
||||
switch err {
|
||||
case nil:
|
||||
s.markSetSize = bytesToInt64(bs)
|
||||
@@ -775,7 +514,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
|
||||
}
|
||||
|
||||
// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
|
||||
bs, err = s.ds.Get(s.ctx, compactionIndexKey)
|
||||
bs, err = s.ds.Get(compactionIndexKey)
|
||||
switch err {
|
||||
case nil:
|
||||
s.compactionIndex = bytesToInt64(bs)
|
||||
@@ -788,7 +527,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
|
||||
return xerrors.Errorf("error loading compaction index: %w", err)
|
||||
}
|
||||
|
||||
log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch.Load())
|
||||
log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch)
|
||||
|
||||
if warmup {
|
||||
err = s.warmup(curTs)
|
||||
@@ -797,9 +536,6 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
|
||||
}
|
||||
}
|
||||
|
||||
// spawn the reifier
|
||||
go s.reifyOrchestrator()
|
||||
|
||||
// watch the chain
|
||||
chain.SubscribeHeadChanges(s.HeadChange)
|
||||
|
||||
@@ -820,24 +556,12 @@ func (s *SplitStore) Close() error {
|
||||
}
|
||||
|
||||
if atomic.LoadInt32(&s.compacting) == 1 {
|
||||
s.txnSyncMx.Lock()
|
||||
s.txnSync = true
|
||||
s.txnSyncCond.Broadcast()
|
||||
s.txnSyncMx.Unlock()
|
||||
|
||||
s.chainSyncMx.Lock()
|
||||
s.chainSyncFinished = true
|
||||
s.chainSyncCond.Broadcast()
|
||||
s.chainSyncMx.Unlock()
|
||||
|
||||
log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
|
||||
for atomic.LoadInt32(&s.compacting) == 1 {
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
s.reifyCond.Broadcast()
|
||||
s.reifyWorkers.Wait()
|
||||
s.cancel()
|
||||
return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
|
||||
}
|
||||
@@ -852,10 +576,5 @@ func (s *SplitStore) checkClosing() error {
|
||||
|
||||
func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
|
||||
s.baseEpoch = epoch
|
||||
return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
|
||||
}
|
||||
|
||||
func (s *SplitStore) setPruneEpoch(epoch abi.ChainEpoch) error {
|
||||
s.pruneEpoch = epoch
|
||||
return s.ds.Put(s.ctx, pruneEpochKey, epochToBytes(epoch))
|
||||
return s.ds.Put(baseEpochKey, epochToBytes(epoch))
|
||||
}
|
||||
|
||||
@@ -4,13 +4,13 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
@@ -25,7 +25,6 @@ func (s *SplitStore) Check() error {
|
||||
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
|
||||
return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
|
||||
}
|
||||
s.compactType = check
|
||||
|
||||
if s.compactionIndex == 0 {
|
||||
atomic.StoreInt32(&s.compacting, 0)
|
||||
@@ -68,10 +67,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
}
|
||||
defer output.Close() //nolint:errcheck
|
||||
|
||||
var mx sync.Mutex
|
||||
write := func(format string, args ...interface{}) {
|
||||
mx.Lock()
|
||||
defer mx.Unlock()
|
||||
_, err := fmt.Fprintf(output, format+"\n", args...)
|
||||
if err != nil {
|
||||
log.Warnf("error writing check output: %s", err)
|
||||
@@ -86,22 +82,14 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
write("compaction index: %d", s.compactionIndex)
|
||||
write("--")
|
||||
|
||||
coldCnt := new(int64)
|
||||
missingCnt := new(int64)
|
||||
|
||||
visitor, err := s.markSetEnv.New("check", 0)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating visitor: %w", err)
|
||||
}
|
||||
defer visitor.Close() //nolint
|
||||
|
||||
size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
|
||||
var coldCnt, missingCnt int64
|
||||
err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
has, err := s.hot.Has(s.ctx, c)
|
||||
has, err := s.hot.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking hotstore: %w", err)
|
||||
}
|
||||
@@ -110,22 +98,22 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
has, err = s.cold.Has(s.ctx, c)
|
||||
has, err = s.cold.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking coldstore: %w", err)
|
||||
}
|
||||
|
||||
if has {
|
||||
atomic.AddInt64(coldCnt, 1)
|
||||
coldCnt++
|
||||
write("cold object reference: %s", c)
|
||||
} else {
|
||||
atomic.AddInt64(missingCnt, 1)
|
||||
missingCnt++
|
||||
write("missing object reference: %s", c)
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
return nil
|
||||
}, func(cid.Cid) error { return nil })
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
err = xerrors.Errorf("error walking chain: %w", err)
|
||||
@@ -133,9 +121,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size)
|
||||
log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
|
||||
write("--")
|
||||
write("cold: %d missing: %d", *coldCnt, *missingCnt)
|
||||
write("cold: %d missing: %d", coldCnt, missingCnt)
|
||||
write("DONE")
|
||||
|
||||
return nil
|
||||
@@ -145,10 +133,8 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
|
||||
func (s *SplitStore) Info() map[string]interface{} {
|
||||
info := make(map[string]interface{})
|
||||
info["base epoch"] = s.baseEpoch
|
||||
info["warmup epoch"] = s.warmupEpoch.Load()
|
||||
info["warmup epoch"] = s.warmupEpoch
|
||||
info["compactions"] = s.compactionIndex
|
||||
info["prunes"] = s.pruneIndex
|
||||
info["compacting"] = s.compacting == 1
|
||||
|
||||
sizer, ok := s.hot.(bstore.BlockstoreSize)
|
||||
if ok {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,8 +5,7 @@ import (
|
||||
"errors"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
@@ -21,29 +20,28 @@ func (s *SplitStore) Expose() bstore.Blockstore {
|
||||
return &exposedSplitStore{s: s}
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) DeleteBlock(_ context.Context, _ cid.Cid) error {
|
||||
func (es *exposedSplitStore) DeleteBlock(_ cid.Cid) error {
|
||||
return errors.New("DeleteBlock: operation not supported")
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) DeleteMany(_ context.Context, _ []cid.Cid) error {
|
||||
func (es *exposedSplitStore) DeleteMany(_ []cid.Cid) error {
|
||||
return errors.New("DeleteMany: operation not supported")
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
|
||||
func (es *exposedSplitStore) Has(c cid.Cid) (bool, error) {
|
||||
if isIdentiyCid(c) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
has, err := es.s.hot.Has(ctx, c)
|
||||
has, err := es.s.hot.Has(c)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
}
|
||||
|
||||
return es.s.cold.Has(ctx, c)
|
||||
return es.s.cold.Has(c)
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
|
||||
|
||||
func (es *exposedSplitStore) Get(c cid.Cid) (blocks.Block, error) {
|
||||
if isIdentiyCid(c) {
|
||||
data, err := decodeIdentityCid(c)
|
||||
if err != nil {
|
||||
@@ -53,14 +51,16 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
|
||||
return blocks.NewBlockWithCid(data, c)
|
||||
}
|
||||
|
||||
blk, err := es.s.hot.Get(ctx, c)
|
||||
if ipld.IsNotFound(err) {
|
||||
return es.s.cold.Get(ctx, c)
|
||||
blk, err := es.s.hot.Get(c)
|
||||
switch err {
|
||||
case bstore.ErrNotFound:
|
||||
return es.s.cold.Get(c)
|
||||
default:
|
||||
return blk, err
|
||||
}
|
||||
return blk, err
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
|
||||
func (es *exposedSplitStore) GetSize(c cid.Cid) (int, error) {
|
||||
if isIdentiyCid(c) {
|
||||
data, err := decodeIdentityCid(c)
|
||||
if err != nil {
|
||||
@@ -70,23 +70,21 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
size, err := es.s.hot.GetSize(ctx, c)
|
||||
if ipld.IsNotFound(err) {
|
||||
return es.s.cold.GetSize(ctx, c)
|
||||
size, err := es.s.hot.GetSize(c)
|
||||
switch err {
|
||||
case bstore.ErrNotFound:
|
||||
return es.s.cold.GetSize(c)
|
||||
default:
|
||||
return size, err
|
||||
}
|
||||
return size, err
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) Flush(ctx context.Context) error {
|
||||
return es.s.Flush(ctx)
|
||||
func (es *exposedSplitStore) Put(blk blocks.Block) error {
|
||||
return es.s.Put(blk)
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
|
||||
return es.s.Put(ctx, blk)
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
|
||||
return es.s.PutMany(ctx, blks)
|
||||
func (es *exposedSplitStore) PutMany(blks []blocks.Block) error {
|
||||
return es.s.PutMany(blks)
|
||||
}
|
||||
|
||||
func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
@@ -95,7 +93,7 @@ func (es *exposedSplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, e
|
||||
|
||||
func (es *exposedSplitStore) HashOnRead(enabled bool) {}
|
||||
|
||||
func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte) error) error {
|
||||
func (es *exposedSplitStore) View(c cid.Cid, f func([]byte) error) error {
|
||||
if isIdentiyCid(c) {
|
||||
data, err := decodeIdentityCid(c)
|
||||
if err != nil {
|
||||
@@ -105,10 +103,12 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
|
||||
return f(data)
|
||||
}
|
||||
|
||||
err := es.s.hot.View(ctx, c, f)
|
||||
if ipld.IsNotFound(err) {
|
||||
return es.s.cold.View(ctx, c, f)
|
||||
}
|
||||
err := es.s.hot.View(c, f)
|
||||
switch err {
|
||||
case bstore.ErrNotFound:
|
||||
return es.s.cold.View(c, f)
|
||||
|
||||
return err
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,96 +7,29 @@ import (
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
)
|
||||
|
||||
const (
|
||||
// Fraction of garbage in badger vlog for online GC traversal to collect garbage
|
||||
AggressiveOnlineGCThreshold = 0.0001
|
||||
)
|
||||
|
||||
func (s *SplitStore) gcHotAfterCompaction() {
|
||||
// Measure hotstore size, determine if we should do full GC, determine if we can do full GC.
|
||||
// We should do full GC if
|
||||
// FullGCFrequency is specified and compaction index matches frequency
|
||||
// OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target
|
||||
// We can do full if
|
||||
// HotstoreMaxSpaceTarget is not specified
|
||||
// OR total moving space would not exceed 50 GB below target
|
||||
//
|
||||
// a) If we should not do full GC => online GC
|
||||
// b) If we should do full GC and can => moving GC
|
||||
// c) If we should do full GC and can't => aggressive online GC
|
||||
getSize := func() int64 {
|
||||
sizer, ok := s.hot.(bstore.BlockstoreSize)
|
||||
if ok {
|
||||
size, err := sizer.Size()
|
||||
if err != nil {
|
||||
log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err)
|
||||
return 0
|
||||
}
|
||||
return size
|
||||
}
|
||||
log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not")
|
||||
return 0
|
||||
}
|
||||
hotSize := getSize()
|
||||
|
||||
copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk
|
||||
shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold)
|
||||
shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0
|
||||
shouldDoFull := shouldTarget || shouldFreq
|
||||
canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer)
|
||||
log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk)
|
||||
log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull)
|
||||
|
||||
func (s *SplitStore) gcHotstore() {
|
||||
var opts []bstore.BlockstoreGCOption
|
||||
if shouldDoFull && canDoFull {
|
||||
if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
|
||||
opts = append(opts, bstore.WithFullGC(true))
|
||||
} else if shouldDoFull && !canDoFull {
|
||||
log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget)
|
||||
log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`")
|
||||
log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`")
|
||||
|
||||
opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold))
|
||||
}
|
||||
|
||||
if err := s.gcBlockstore(s.hot, opts); err != nil {
|
||||
log.Warnf("error garbage collecting hostore: %s", err)
|
||||
}
|
||||
log.Infof("measured hot store size after GC: %d", getSize())
|
||||
}
|
||||
|
||||
func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
|
||||
if err := s.checkYield(); err != nil {
|
||||
return err
|
||||
}
|
||||
if gc, ok := b.(bstore.BlockstoreGC); ok {
|
||||
log.Info("garbage collecting blockstore")
|
||||
startGC := time.Now()
|
||||
|
||||
opts = append(opts, bstore.WithCheckFreq(90*time.Second))
|
||||
opts = append(opts, bstore.WithCheck(s.checkYield))
|
||||
if err := gc.CollectGarbage(s.ctx, opts...); err != nil {
|
||||
if err := gc.CollectGarbage(opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infow("garbage collecting blockstore done", "took", time.Since(startGC))
|
||||
log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
|
||||
}
|
||||
|
||||
func (s *SplitStore) gcBlockstoreOnce(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
|
||||
if gc, ok := b.(bstore.BlockstoreGCOnce); ok {
|
||||
log.Debug("gc blockstore once")
|
||||
startGC := time.Now()
|
||||
|
||||
if err := gc.GCOnce(s.ctx, opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugw("gc blockstore once done", "took", time.Since(startGC))
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("blockstore doesn't support gc once: %T", b)
|
||||
}
|
||||
|
||||
@@ -1,591 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"go.opencensus.io/stats"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
// PruneOnline is a prune option that instructs PruneChain to use online gc for reclaiming space;
|
||||
// there is no value associated with this option.
|
||||
PruneOnlineGC = "splitstore.PruneOnlineGC"
|
||||
|
||||
// PruneMoving is a prune option that instructs PruneChain to use moving gc for reclaiming space;
|
||||
// the value associated with this option is the path of the new coldstore.
|
||||
PruneMovingGC = "splitstore.PruneMovingGC"
|
||||
|
||||
// PruneRetainState is a prune option that instructs PruneChain as to how many finalities worth
|
||||
// of state to retain in the coldstore.
|
||||
// The value is an integer:
|
||||
// - if it is -1 then all state objects reachable from the chain will be retained in the coldstore.
|
||||
// this is useful for garbage collecting side-chains and other garbage in archival nodes.
|
||||
// This is the (safe) default.
|
||||
// - if it is 0 then no state objects that are unreachable within the compaction boundary will
|
||||
// be retained in the coldstore.
|
||||
// - if it is a positive integer, then it's the number of finalities past the compaction boundary
|
||||
// for which chain-reachable state objects are retained.
|
||||
PruneRetainState = "splitstore.PruneRetainState"
|
||||
|
||||
// PruneThreshold is the number of epochs that need to have elapsed
|
||||
// from the previously pruned epoch to trigger a new prune
|
||||
PruneThreshold = 7 * build.Finality
|
||||
)
|
||||
|
||||
// GCHotstore runs online GC on the chain state in the hotstore according the to options specified
|
||||
func (s *SplitStore) GCHotStore(opts api.HotGCOpts) error {
|
||||
if opts.Moving {
|
||||
gcOpts := []bstore.BlockstoreGCOption{bstore.WithFullGC(true)}
|
||||
return s.gcBlockstore(s.hot, gcOpts)
|
||||
}
|
||||
|
||||
gcOpts := []bstore.BlockstoreGCOption{bstore.WithThreshold(opts.Threshold)}
|
||||
var err error
|
||||
if opts.Periodic {
|
||||
err = s.gcBlockstore(s.hot, gcOpts)
|
||||
} else {
|
||||
err = s.gcBlockstoreOnce(s.hot, gcOpts)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// PruneChain instructs the SplitStore to prune chain state in the coldstore, according to the
|
||||
// options specified.
|
||||
func (s *SplitStore) PruneChain(opts api.PruneOpts) error {
|
||||
retainState := opts.RetainState
|
||||
|
||||
var gcOpts []bstore.BlockstoreGCOption
|
||||
if opts.MovingGC {
|
||||
gcOpts = append(gcOpts, bstore.WithFullGC(true))
|
||||
}
|
||||
doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
|
||||
|
||||
var retainStateP func(int64) bool
|
||||
switch {
|
||||
case retainState > 0:
|
||||
retainStateP = func(depth int64) bool {
|
||||
return depth <= int64(CompactionBoundary)+retainState*int64(build.Finality)
|
||||
}
|
||||
case retainState < 0:
|
||||
retainStateP = func(_ int64) bool { return true }
|
||||
default:
|
||||
retainStateP = func(depth int64) bool {
|
||||
return depth <= int64(CompactionBoundary)
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := s.cold.(bstore.BlockstoreIterator); !ok {
|
||||
return xerrors.Errorf("coldstore does not support efficient iteration")
|
||||
}
|
||||
|
||||
return s.pruneChain(retainStateP, doGC)
|
||||
}
|
||||
|
||||
func (s *SplitStore) pruneChain(retainStateP func(int64) bool, doGC func() error) error {
|
||||
// inhibit compaction while we are setting up
|
||||
s.headChangeMx.Lock()
|
||||
defer s.headChangeMx.Unlock()
|
||||
|
||||
// take the compaction lock; fail if there is a compaction in progress
|
||||
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
|
||||
return xerrors.Errorf("compaction, prune or warmup in progress")
|
||||
}
|
||||
|
||||
// check if we are actually closing first
|
||||
if atomic.LoadInt32(&s.closing) == 1 {
|
||||
atomic.StoreInt32(&s.compacting, 0)
|
||||
return errClosing
|
||||
}
|
||||
|
||||
// ensure that we have compacted at least once
|
||||
if s.compactionIndex == 0 {
|
||||
atomic.StoreInt32(&s.compacting, 0)
|
||||
return xerrors.Errorf("splitstore has not compacted yet")
|
||||
}
|
||||
|
||||
// get the current tipset
|
||||
curTs := s.chain.GetHeaviestTipSet()
|
||||
|
||||
// begin the transaction and go
|
||||
s.beginTxnProtect()
|
||||
s.compactType = cold
|
||||
go func() {
|
||||
defer atomic.StoreInt32(&s.compacting, 0)
|
||||
defer s.endTxnProtect()
|
||||
|
||||
log.Info("pruning splitstore")
|
||||
start := time.Now()
|
||||
|
||||
s.prune(curTs, retainStateP, doGC)
|
||||
|
||||
log.Infow("prune done", "took", time.Since(start))
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) prune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) {
|
||||
log.Debug("waiting for active views to complete")
|
||||
start := time.Now()
|
||||
s.viewWait()
|
||||
log.Debugw("waiting for active views done", "took", time.Since(start))
|
||||
|
||||
err := s.doPrune(curTs, retainStateP, doGC)
|
||||
if err != nil {
|
||||
log.Errorf("PRUNE ERROR: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) doPrune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) error {
|
||||
currentEpoch := curTs.Height()
|
||||
boundaryEpoch := currentEpoch - CompactionBoundary
|
||||
|
||||
log.Infow("running prune", "currentEpoch", currentEpoch, "pruneEpoch", s.pruneEpoch)
|
||||
|
||||
markSet, err := s.markSetEnv.New("live", s.markSetSize)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating mark set: %w", err)
|
||||
}
|
||||
defer markSet.Close() //nolint:errcheck
|
||||
defer s.debug.Flush()
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 0. track all protected references at beginning of compaction; anything added later should
|
||||
// be transactionally protected by the write
|
||||
log.Info("protecting references with registered protectors")
|
||||
err = s.applyProtectors()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1. mark reachable objects by walking the chain from the current epoch; we keep all messages
|
||||
// and chain headers; state and reciepts are retained only if it is within retention policy scope
|
||||
log.Info("marking reachable objects")
|
||||
startMark := time.Now()
|
||||
|
||||
count := new(int64)
|
||||
err = s.walkChainDeep(curTs, retainStateP,
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
mark, err := markSet.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking markset: %w", err)
|
||||
}
|
||||
|
||||
if mark {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
atomic.AddInt64(count, 1)
|
||||
return markSet.Mark(c)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error marking: %w", err)
|
||||
}
|
||||
|
||||
log.Infow("marking done", "took", time.Since(startMark), "marked", count)
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1.1 protect transactional refs
|
||||
err = s.protectTxnRefs(markSet)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error protecting transactional refs: %w", err)
|
||||
}
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. iterate through the coldstore to collect dead objects
|
||||
log.Info("collecting dead objects")
|
||||
startCollect := time.Now()
|
||||
|
||||
deadw, err := NewColdSetWriter(s.discardSetPath())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating coldset: %w", err)
|
||||
}
|
||||
defer deadw.Close() //nolint:errcheck
|
||||
|
||||
// some stats for logging
|
||||
var liveCnt, deadCnt int
|
||||
|
||||
err = s.cold.(bstore.BlockstoreIterator).ForEachKey(func(c cid.Cid) error {
|
||||
// was it marked?
|
||||
mark, err := markSet.Has(c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking mark set for %s: %w", c, err)
|
||||
}
|
||||
|
||||
if mark {
|
||||
liveCnt++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Note: a possible optimization here is to also purge objects that are in the hotstore
|
||||
// but this needs special care not to purge genesis state, so we don't bother (yet)
|
||||
|
||||
// it's dead in the coldstore, mark it as candidate for purge
|
||||
|
||||
if err := deadw.Write(c); err != nil {
|
||||
return xerrors.Errorf("error writing cid to coldstore: %w", err)
|
||||
}
|
||||
deadCnt++
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error dead objects: %w", err)
|
||||
}
|
||||
|
||||
if err := deadw.Close(); err != nil {
|
||||
return xerrors.Errorf("error closing deadset: %w", err)
|
||||
}
|
||||
|
||||
stats.Record(s.ctx, metrics.SplitstoreCompactionDead.M(int64(deadCnt)))
|
||||
|
||||
log.Infow("dead collection done", "took", time.Since(startCollect))
|
||||
log.Infow("prune stats", "live", liveCnt, "dead", deadCnt)
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// now that we have collected dead objects, check for missing references from transactional i/o
|
||||
// this is carried over from hot compaction for completeness
|
||||
s.waitForMissingRefs(markSet)
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
deadr, err := NewColdSetReader(s.discardSetPath())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error opening deadset: %w", err)
|
||||
}
|
||||
defer deadr.Close() //nolint:errcheck
|
||||
|
||||
// 3. Purge dead objects with checkpointing for recovery.
|
||||
// This is the critical section of prune, whereby any dead object not in the markSet is
|
||||
// considered already deleted.
|
||||
// We delete dead objects in batches, holding the transaction lock, where we check the markSet
|
||||
// again for new references created by the caller.
|
||||
// After each batch we write a checkpoint to disk; if the process is interrupted before completion
|
||||
// the process will continue from the checkpoint in the next recovery.
|
||||
if err := s.beginCriticalSection(markSet); err != nil {
|
||||
return xerrors.Errorf("error beginning critical section: %w", err)
|
||||
}
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
checkpoint, err := NewCheckpoint(s.pruneCheckpointPath())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating checkpoint: %w", err)
|
||||
}
|
||||
defer checkpoint.Close() //nolint:errcheck
|
||||
|
||||
log.Info("purging dead objects from the coldstore")
|
||||
startPurge := time.Now()
|
||||
err = s.purge(deadr, checkpoint, markSet)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error purging dead objects: %w", err)
|
||||
}
|
||||
log.Infow("purging dead objects from coldstore done", "took", time.Since(startPurge))
|
||||
|
||||
s.endCriticalSection()
|
||||
|
||||
if err := checkpoint.Close(); err != nil {
|
||||
log.Warnf("error closing checkpoint: %s", err)
|
||||
}
|
||||
if err := os.Remove(s.pruneCheckpointPath()); err != nil {
|
||||
log.Warnf("error removing checkpoint: %s", err)
|
||||
}
|
||||
if err := deadr.Close(); err != nil {
|
||||
log.Warnf("error closing discard set: %s", err)
|
||||
}
|
||||
if err := os.Remove(s.discardSetPath()); err != nil {
|
||||
log.Warnf("error removing discard set: %s", err)
|
||||
}
|
||||
|
||||
// we are done; do some housekeeping
|
||||
s.endTxnProtect()
|
||||
err = doGC()
|
||||
if err != nil {
|
||||
log.Warnf("error garbage collecting cold store: %s", err)
|
||||
}
|
||||
|
||||
if err := s.setPruneEpoch(boundaryEpoch); err != nil {
|
||||
return xerrors.Errorf("error saving prune base epoch: %w", err)
|
||||
}
|
||||
|
||||
s.pruneIndex++
|
||||
err = s.ds.Put(s.ctx, pruneIndexKey, int64ToBytes(s.pruneIndex))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error saving prune index: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SplitStore) completePrune() error {
|
||||
checkpoint, last, err := OpenCheckpoint(s.pruneCheckpointPath())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error opening checkpoint: %w", err)
|
||||
}
|
||||
defer checkpoint.Close() //nolint:errcheck
|
||||
|
||||
deadr, err := NewColdSetReader(s.discardSetPath())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error opening deadset: %w", err)
|
||||
}
|
||||
defer deadr.Close() //nolint:errcheck
|
||||
|
||||
markSet, err := s.markSetEnv.Recover("live")
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error recovering markset: %w", err)
|
||||
}
|
||||
defer markSet.Close() //nolint:errcheck
|
||||
|
||||
// PURGE!
|
||||
s.compactType = cold
|
||||
log.Info("purging dead objects from the coldstore")
|
||||
startPurge := time.Now()
|
||||
err = s.completePurge(deadr, checkpoint, last, markSet)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error purgin dead objects: %w", err)
|
||||
}
|
||||
log.Infow("purging dead objects from the coldstore done", "took", time.Since(startPurge))
|
||||
|
||||
markSet.EndCriticalSection()
|
||||
s.compactType = none
|
||||
|
||||
if err := checkpoint.Close(); err != nil {
|
||||
log.Warnf("error closing checkpoint: %s", err)
|
||||
}
|
||||
if err := os.Remove(s.pruneCheckpointPath()); err != nil {
|
||||
log.Warnf("error removing checkpoint: %s", err)
|
||||
}
|
||||
if err := deadr.Close(); err != nil {
|
||||
log.Warnf("error closing deadset: %s", err)
|
||||
}
|
||||
if err := os.Remove(s.discardSetPath()); err != nil {
|
||||
log.Warnf("error removing deadset: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// like walkChain but peforms a deep walk, using parallel walking with walkObjectLax,
|
||||
// whereby all extant messages are retained and state roots are retained if they satisfy
|
||||
// the given predicate.
|
||||
// missing references are ignored, as we expect to have plenty for snapshot syncs.
|
||||
func (s *SplitStore) walkChainDeep(ts *types.TipSet, retainStateP func(int64) bool,
|
||||
f func(cid.Cid) error) error {
|
||||
visited := cid.NewSet()
|
||||
toWalk := ts.Cids()
|
||||
walkCnt := 0
|
||||
|
||||
workers := runtime.NumCPU() / 2
|
||||
if workers < 2 {
|
||||
workers = 2
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
workch := make(chan cid.Cid, 16*workers)
|
||||
errch := make(chan error, workers)
|
||||
|
||||
var once sync.Once
|
||||
defer once.Do(func() { close(workch) })
|
||||
|
||||
push := func(c cid.Cid) error {
|
||||
if !visited.Visit(c) {
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case workch <- c:
|
||||
return nil
|
||||
case err := <-errch:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
worker := func() {
|
||||
defer wg.Done()
|
||||
for c := range workch {
|
||||
err := s.walkObjectLax(c, f)
|
||||
if err != nil {
|
||||
errch <- xerrors.Errorf("error walking object (cid: %s): %w", c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go worker()
|
||||
}
|
||||
|
||||
baseEpoch := ts.Height()
|
||||
minEpoch := baseEpoch // for progress report
|
||||
log.Infof("walking at epoch %d", minEpoch)
|
||||
|
||||
walkBlock := func(c cid.Cid) error {
|
||||
if !visited.Visit(c) {
|
||||
return nil
|
||||
}
|
||||
|
||||
walkCnt++
|
||||
|
||||
if err := f(c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var hdr types.BlockHeader
|
||||
err := s.view(c, func(data []byte) error {
|
||||
return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
|
||||
}
|
||||
|
||||
if hdr.Height < minEpoch {
|
||||
minEpoch = hdr.Height
|
||||
if minEpoch%10_000 == 0 {
|
||||
log.Infof("walking at epoch %d (walked: %d)", minEpoch, walkCnt)
|
||||
}
|
||||
}
|
||||
|
||||
depth := int64(baseEpoch - hdr.Height)
|
||||
retainState := retainStateP(depth)
|
||||
|
||||
if hdr.Height > 0 {
|
||||
if err := push(hdr.Messages); err != nil {
|
||||
return err
|
||||
}
|
||||
if retainState {
|
||||
if err := push(hdr.ParentMessageReceipts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if retainState || hdr.Height == 0 {
|
||||
if err := push(hdr.ParentStateRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if hdr.Height > 0 {
|
||||
toWalk = append(toWalk, hdr.Parents...)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
for len(toWalk) > 0 {
|
||||
// walking can take a while, so check this with every opportunity
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-errch:
|
||||
return err
|
||||
default:
|
||||
}
|
||||
|
||||
walking := toWalk
|
||||
toWalk = nil
|
||||
for _, c := range walking {
|
||||
if err := walkBlock(c); err != nil {
|
||||
return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
once.Do(func() { close(workch) })
|
||||
wg.Wait()
|
||||
select {
|
||||
case err := <-errch:
|
||||
return err
|
||||
default:
|
||||
}
|
||||
|
||||
log.Infow("chain walk done", "walked", walkCnt)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// like walkObject but treats missing references laxly; faster version of walkObjectIncomplete
|
||||
// without an occurs check.
|
||||
func (s *SplitStore) walkObjectLax(c cid.Cid, f func(cid.Cid) error) error {
|
||||
if err := f(c); err != nil {
|
||||
if err == errStopWalk {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if c.Prefix().Codec != cid.DagCBOR {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check this before recursing
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var links []cid.Cid
|
||||
err := s.view(c, func(data []byte) error {
|
||||
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
|
||||
links = append(links, c)
|
||||
})
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if ipld.IsNotFound(err) { // not a problem for deep walks
|
||||
return nil
|
||||
}
|
||||
|
||||
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
|
||||
}
|
||||
|
||||
for _, c := range links {
|
||||
err := s.walkObjectLax(c, f)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
var (
|
||||
errReifyLimit = errors.New("reification limit reached")
|
||||
ReifyLimit = 16384
|
||||
)
|
||||
|
||||
func (s *SplitStore) reifyColdObject(c cid.Cid) {
|
||||
if !s.isWarm() {
|
||||
return
|
||||
}
|
||||
|
||||
if isUnitaryObject(c) {
|
||||
return
|
||||
}
|
||||
|
||||
s.reifyMx.Lock()
|
||||
defer s.reifyMx.Unlock()
|
||||
|
||||
_, ok := s.reifyInProgress[c]
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
|
||||
s.reifyPend[c] = struct{}{}
|
||||
s.reifyCond.Broadcast()
|
||||
}
|
||||
|
||||
func (s *SplitStore) reifyOrchestrator() {
|
||||
workers := runtime.NumCPU() / 4
|
||||
if workers < 2 {
|
||||
workers = 2
|
||||
}
|
||||
|
||||
workch := make(chan cid.Cid, workers)
|
||||
defer close(workch)
|
||||
|
||||
for i := 0; i < workers; i++ {
|
||||
s.reifyWorkers.Add(1)
|
||||
go s.reifyWorker(workch)
|
||||
}
|
||||
|
||||
for {
|
||||
s.reifyMx.Lock()
|
||||
for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
|
||||
s.reifyCond.Wait()
|
||||
}
|
||||
|
||||
if atomic.LoadInt32(&s.closing) != 0 {
|
||||
s.reifyMx.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
reifyPend := s.reifyPend
|
||||
s.reifyPend = make(map[cid.Cid]struct{})
|
||||
s.reifyMx.Unlock()
|
||||
|
||||
for c := range reifyPend {
|
||||
select {
|
||||
case workch <- c:
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
|
||||
defer s.reifyWorkers.Done()
|
||||
for c := range workch {
|
||||
s.doReify(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SplitStore) doReify(c cid.Cid) {
|
||||
var toreify, toforget []cid.Cid
|
||||
|
||||
defer func() {
|
||||
s.reifyMx.Lock()
|
||||
defer s.reifyMx.Unlock()
|
||||
|
||||
for _, c := range toreify {
|
||||
delete(s.reifyInProgress, c)
|
||||
}
|
||||
for _, c := range toforget {
|
||||
delete(s.reifyInProgress, c)
|
||||
}
|
||||
}()
|
||||
|
||||
s.txnLk.RLock()
|
||||
defer s.txnLk.RUnlock()
|
||||
|
||||
count := 0
|
||||
_, err := s.walkObjectIncomplete(c, newTmpVisitor(),
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
count++
|
||||
if count > ReifyLimit {
|
||||
return errReifyLimit
|
||||
}
|
||||
|
||||
s.reifyMx.Lock()
|
||||
_, inProgress := s.reifyInProgress[c]
|
||||
if !inProgress {
|
||||
s.reifyInProgress[c] = struct{}{}
|
||||
}
|
||||
s.reifyMx.Unlock()
|
||||
|
||||
if inProgress {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
has, err := s.hot.Has(s.ctx, c)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error checking hotstore: %w", err)
|
||||
}
|
||||
|
||||
// All reified blocks are tracked at reification start
|
||||
if has {
|
||||
toforget = append(toforget, c)
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
toreify = append(toreify, c)
|
||||
return nil
|
||||
},
|
||||
func(missing cid.Cid) error {
|
||||
log.Warnf("missing reference while reifying %s: %s", c, missing)
|
||||
return errStopWalk
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, errReifyLimit) {
|
||||
log.Debug("reification aborted; reify limit reached")
|
||||
return
|
||||
}
|
||||
|
||||
log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
|
||||
|
||||
// this should not get too big, maybe some 100s of objects.
|
||||
batch := make([]blocks.Block, 0, len(toreify))
|
||||
for _, c := range toreify {
|
||||
blk, err := s.cold.Get(s.ctx, c)
|
||||
if err != nil {
|
||||
log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.checkClosing(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
batch = append(batch, blk)
|
||||
}
|
||||
|
||||
if len(batch) > 0 {
|
||||
err = s.hot.PutMany(s.ctx, batch)
|
||||
if err != nil {
|
||||
log.Warnf("error reifying cold object (cid: %s): %s", c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,44 +1,35 @@
|
||||
// stm: #unit
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-datastore"
|
||||
dssync "github.com/ipfs/go-datastore/sync"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/chain/types/mock"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
datastore "github.com/ipfs/go-datastore"
|
||||
dssync "github.com/ipfs/go-datastore/sync"
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
CompactionThreshold = 5
|
||||
CompactionBoundary = 2
|
||||
WarmupBoundary = 0
|
||||
SyncWaitTime = time.Millisecond
|
||||
logging.SetLogLevel("splitstore", "DEBUG")
|
||||
}
|
||||
|
||||
func testSplitStore(t *testing.T, cfg *Config) {
|
||||
ctx := context.Background()
|
||||
chain := &mockChain{t: t}
|
||||
fmt.Printf("Config: %v\n", cfg)
|
||||
|
||||
// the myriads of stores
|
||||
ds := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
@@ -47,7 +38,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
|
||||
// this is necessary to avoid the garbage mock puts in the blocks
|
||||
garbage := blocks.NewBlock([]byte{1, 2, 3})
|
||||
err := cold.Put(ctx, garbage)
|
||||
err := cold.Put(garbage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -68,29 +59,27 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = cold.Put(ctx, blk)
|
||||
err = cold.Put(blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// create a garbage block that is protected with a rgistered protector
|
||||
protected := blocks.NewBlock([]byte("protected!"))
|
||||
err = hot.Put(ctx, protected)
|
||||
err = hot.Put(protected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// and another one that is not protected
|
||||
unprotected := blocks.NewBlock([]byte("unprotected!"))
|
||||
err = hot.Put(ctx, unprotected)
|
||||
err = hot.Put(unprotected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
path := t.TempDir()
|
||||
|
||||
// open the splitstore
|
||||
ss, err := Open(path, ds, hot, cold, cfg)
|
||||
ss, err := Open("", ds, hot, cold, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,7 +90,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
return protect(protected.Cid())
|
||||
})
|
||||
|
||||
err = ss.Start(chain, nil)
|
||||
err = ss.Start(chain)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -119,11 +108,11 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(ctx, stateRoot)
|
||||
err = ss.Put(stateRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(ctx, sblk)
|
||||
err = ss.Put(sblk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -134,10 +123,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
}
|
||||
|
||||
waitForCompaction := func() {
|
||||
ss.txnSyncMx.Lock()
|
||||
ss.txnSync = true
|
||||
ss.txnSyncCond.Broadcast()
|
||||
ss.txnSyncMx.Unlock()
|
||||
for atomic.LoadInt32(&ss.compacting) == 1 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
@@ -190,7 +175,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
}
|
||||
|
||||
// ensure our protected block is still there
|
||||
has, err := hot.Has(ctx, protected.Cid())
|
||||
has, err := hot.Has(protected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -200,7 +185,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
}
|
||||
|
||||
// ensure our unprotected block is in the coldstore now
|
||||
has, err = hot.Has(ctx, unprotected.Cid())
|
||||
has, err = hot.Has(unprotected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -209,7 +194,7 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
t.Fatal("unprotected block is still in hotstore")
|
||||
}
|
||||
|
||||
has, err = cold.Has(ctx, unprotected.Cid())
|
||||
has, err = cold.Has(unprotected.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -223,381 +208,16 @@ func testSplitStore(t *testing.T, cfg *Config) {
|
||||
}
|
||||
|
||||
func TestSplitStoreCompaction(t *testing.T) {
|
||||
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
testSplitStore(t, &Config{MarkSetType: "map", UniversalColdBlocks: true})
|
||||
testSplitStore(t, &Config{MarkSetType: "map"})
|
||||
}
|
||||
|
||||
func TestSplitStoreCompactionWithBadger(t *testing.T) {
|
||||
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
bs := badgerMarkSetBatchSize
|
||||
badgerMarkSetBatchSize = 1
|
||||
t.Cleanup(func() {
|
||||
badgerMarkSetBatchSize = bs
|
||||
})
|
||||
testSplitStore(t, &Config{MarkSetType: "badger", UniversalColdBlocks: true})
|
||||
}
|
||||
|
||||
func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
|
||||
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
|
||||
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
|
||||
ctx := context.Background()
|
||||
chain := &mockChain{t: t}
|
||||
|
||||
// the myriads of stores
|
||||
ds := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
hot := newMockStore()
|
||||
cold := newMockStore()
|
||||
|
||||
// this is necessary to avoid the garbage mock puts in the blocks
|
||||
garbage := blocks.NewBlock([]byte{1, 2, 3})
|
||||
err := cold.Put(ctx, garbage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// genesis
|
||||
genBlock := mock.MkBlock(nil, 0, 0)
|
||||
genBlock.Messages = garbage.Cid()
|
||||
genBlock.ParentMessageReceipts = garbage.Cid()
|
||||
genBlock.ParentStateRoot = garbage.Cid()
|
||||
genBlock.Timestamp = uint64(time.Now().Unix())
|
||||
|
||||
genTs := mock.TipSet(genBlock)
|
||||
chain.push(genTs)
|
||||
|
||||
// put the genesis block to cold store
|
||||
blk, err := genBlock.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = cold.Put(ctx, blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
path := t.TempDir()
|
||||
|
||||
// open the splitstore
|
||||
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ss.Close() //nolint
|
||||
|
||||
// create an upgrade schedule that will suppress compaction during the test
|
||||
upgradeBoundary = 0
|
||||
upgrade := stmgr.Upgrade{
|
||||
Height: 10,
|
||||
PreMigrations: []stmgr.PreMigration{{StartWithin: 10}},
|
||||
}
|
||||
|
||||
err = ss.Start(chain, []stmgr.Upgrade{upgrade})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mkBlock := func(curTs *types.TipSet, i int, stateRoot blocks.Block) *types.TipSet {
|
||||
blk := mock.MkBlock(curTs, uint64(i), uint64(i))
|
||||
|
||||
blk.Messages = garbage.Cid()
|
||||
blk.ParentMessageReceipts = garbage.Cid()
|
||||
blk.ParentStateRoot = stateRoot.Cid()
|
||||
blk.Timestamp = uint64(time.Now().Unix())
|
||||
|
||||
sblk, err := blk.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(ctx, stateRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ss.Put(ctx, sblk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := mock.TipSet(blk)
|
||||
chain.push(ts)
|
||||
|
||||
return ts
|
||||
}
|
||||
|
||||
waitForCompaction := func() {
|
||||
ss.txnSyncMx.Lock()
|
||||
ss.txnSync = true
|
||||
ss.txnSyncCond.Broadcast()
|
||||
ss.txnSyncMx.Unlock()
|
||||
for atomic.LoadInt32(&ss.compacting) == 1 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
curTs := genTs
|
||||
for i := 1; i < 10; i++ {
|
||||
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
|
||||
curTs = mkBlock(curTs, i, stateRoot)
|
||||
waitForCompaction()
|
||||
}
|
||||
|
||||
countBlocks := func(bs blockstore.Blockstore) int {
|
||||
count := 0
|
||||
_ = bs.(blockstore.BlockstoreIterator).ForEachKey(func(_ cid.Cid) error {
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
// we should not have compacted due to suppression and everything should still be hot
|
||||
hotCnt := countBlocks(hot)
|
||||
coldCnt := countBlocks(cold)
|
||||
|
||||
if hotCnt != 20 {
|
||||
t.Errorf("expected %d blocks, but got %d", 20, hotCnt)
|
||||
}
|
||||
|
||||
if coldCnt != 2 {
|
||||
t.Errorf("expected %d blocks, but got %d", 2, coldCnt)
|
||||
}
|
||||
|
||||
// put some more blocks, now we should compact
|
||||
for i := 10; i < 20; i++ {
|
||||
stateRoot := blocks.NewBlock([]byte{byte(i), 3, 3, 7})
|
||||
curTs = mkBlock(curTs, i, stateRoot)
|
||||
waitForCompaction()
|
||||
}
|
||||
|
||||
hotCnt = countBlocks(hot)
|
||||
coldCnt = countBlocks(cold)
|
||||
|
||||
if hotCnt != 24 {
|
||||
t.Errorf("expected %d blocks, but got %d", 24, hotCnt)
|
||||
}
|
||||
|
||||
if coldCnt != 18 {
|
||||
t.Errorf("expected %d blocks, but got %d", 18, coldCnt)
|
||||
}
|
||||
}
|
||||
|
||||
func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
|
||||
ds := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
hot := newMockStore()
|
||||
cold := newMockStore()
|
||||
|
||||
mkRandomBlock := func() blocks.Block {
|
||||
data := make([]byte, 128)
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return blocks.NewBlock(data)
|
||||
}
|
||||
|
||||
block1 := mkRandomBlock()
|
||||
block2 := mkRandomBlock()
|
||||
block3 := mkRandomBlock()
|
||||
|
||||
hdr := mock.MkBlock(nil, 0, 0)
|
||||
hdr.Messages = block1.Cid()
|
||||
hdr.ParentMessageReceipts = block2.Cid()
|
||||
hdr.ParentStateRoot = block3.Cid()
|
||||
block4, err := hdr.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
allBlocks := []blocks.Block{block1, block2, block3, block4}
|
||||
for _, blk := range allBlocks {
|
||||
err := cold.Put(context.Background(), blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
path := t.TempDir()
|
||||
|
||||
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ss.Close() //nolint
|
||||
|
||||
ss.warmupEpoch.Store(1)
|
||||
go ss.reifyOrchestrator()
|
||||
|
||||
waitForReification := func() {
|
||||
for {
|
||||
ss.reifyMx.Lock()
|
||||
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
|
||||
ss.reifyMx.Unlock()
|
||||
|
||||
if ready {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// first access using the standard view
|
||||
err = f(context.Background(), ss, block4.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// nothing should be reified
|
||||
waitForReification()
|
||||
for _, blk := range allBlocks {
|
||||
has, err := hot.Has(context.Background(), blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("block unexpectedly reified")
|
||||
}
|
||||
}
|
||||
|
||||
// now make the hot/reifying view and ensure access reifies
|
||||
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// everything should be reified
|
||||
waitForReification()
|
||||
for i, blk := range allBlocks {
|
||||
has, err := hot.Has(context.Background(), blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !has {
|
||||
t.Fatalf("block%d was not reified", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
|
||||
ds := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
hot := newMockStore()
|
||||
cold := newMockStore()
|
||||
|
||||
mkRandomBlock := func() blocks.Block {
|
||||
data := make([]byte, 128)
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return blocks.NewBlock(data)
|
||||
}
|
||||
|
||||
block1 := mkRandomBlock()
|
||||
block2 := mkRandomBlock()
|
||||
block3 := mkRandomBlock()
|
||||
|
||||
hdr := mock.MkBlock(nil, 0, 0)
|
||||
hdr.Messages = block1.Cid()
|
||||
hdr.ParentMessageReceipts = block2.Cid()
|
||||
hdr.ParentStateRoot = block3.Cid()
|
||||
block4, err := hdr.ToStorageBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
allBlocks := []blocks.Block{block1, block2, block3, block4}
|
||||
for _, blk := range allBlocks {
|
||||
err := cold.Put(context.Background(), blk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
path := t.TempDir()
|
||||
|
||||
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ss.Close() //nolint
|
||||
|
||||
ss.warmupEpoch.Store(1)
|
||||
go ss.reifyOrchestrator()
|
||||
|
||||
waitForReification := func() {
|
||||
for {
|
||||
ss.reifyMx.Lock()
|
||||
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
|
||||
ss.reifyMx.Unlock()
|
||||
|
||||
if ready {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// do a hot access -- nothing should be reified as the limit should be exceeded
|
||||
oldReifyLimit := ReifyLimit
|
||||
ReifyLimit = 2
|
||||
t.Cleanup(func() {
|
||||
ReifyLimit = oldReifyLimit
|
||||
})
|
||||
|
||||
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
waitForReification()
|
||||
|
||||
for _, blk := range allBlocks {
|
||||
has, err := hot.Has(context.Background(), blk.Cid())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if has {
|
||||
t.Fatal("block unexpectedly reified")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSplitStoreReification(t *testing.T) {
|
||||
t.Log("test reification with Has")
|
||||
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
|
||||
_, err := s.Has(ctx, c)
|
||||
return err
|
||||
})
|
||||
t.Log("test reification with Get")
|
||||
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
|
||||
_, err := s.Get(ctx, c)
|
||||
return err
|
||||
})
|
||||
t.Log("test reification with GetSize")
|
||||
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
|
||||
_, err := s.GetSize(ctx, c)
|
||||
return err
|
||||
})
|
||||
t.Log("test reification with View")
|
||||
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
|
||||
return s.View(ctx, c, func(_ []byte) error { return nil })
|
||||
})
|
||||
t.Log("test reification limit")
|
||||
testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
|
||||
_, err := s.Has(ctx, c)
|
||||
return err
|
||||
})
|
||||
testSplitStore(t, &Config{MarkSetType: "badger"})
|
||||
}
|
||||
|
||||
type mockChain struct {
|
||||
@@ -669,43 +289,35 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
|
||||
|
||||
type mockStore struct {
|
||||
mx sync.Mutex
|
||||
set map[string]blocks.Block
|
||||
set map[cid.Cid]blocks.Block
|
||||
}
|
||||
|
||||
func newMockStore() *mockStore {
|
||||
return &mockStore{set: make(map[string]blocks.Block)}
|
||||
return &mockStore{set: make(map[cid.Cid]blocks.Block)}
|
||||
}
|
||||
|
||||
func (b *mockStore) keyOf(c cid.Cid) string {
|
||||
return string(c.Hash())
|
||||
}
|
||||
|
||||
func (b *mockStore) cidOf(k string) cid.Cid {
|
||||
return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
|
||||
}
|
||||
|
||||
func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
|
||||
func (b *mockStore) Has(cid cid.Cid) (bool, error) {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
_, ok := b.set[b.keyOf(cid)]
|
||||
_, ok := b.set[cid]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (b *mockStore) HashOnRead(hor bool) {}
|
||||
|
||||
func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
func (b *mockStore) Get(cid cid.Cid) (blocks.Block, error) {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
|
||||
blk, ok := b.set[b.keyOf(cid)]
|
||||
blk, ok := b.set[cid]
|
||||
if !ok {
|
||||
return nil, ipld.ErrNotFound{Cid: cid}
|
||||
return nil, blockstore.ErrNotFound
|
||||
}
|
||||
return blk, nil
|
||||
}
|
||||
|
||||
func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
blk, err := b.Get(ctx, cid)
|
||||
func (b *mockStore) GetSize(cid cid.Cid) (int, error) {
|
||||
blk, err := b.Get(cid)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -713,52 +325,50 @@ func (b *mockStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
|
||||
return len(blk.RawData()), nil
|
||||
}
|
||||
|
||||
func (b *mockStore) View(ctx context.Context, cid cid.Cid, f func([]byte) error) error {
|
||||
blk, err := b.Get(ctx, cid)
|
||||
func (b *mockStore) View(cid cid.Cid, f func([]byte) error) error {
|
||||
blk, err := b.Get(cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f(blk.RawData())
|
||||
}
|
||||
|
||||
func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
|
||||
func (b *mockStore) Put(blk blocks.Block) error {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
|
||||
b.set[b.keyOf(blk.Cid())] = blk
|
||||
b.set[blk.Cid()] = blk
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
|
||||
func (b *mockStore) PutMany(blks []blocks.Block) error {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
|
||||
for _, blk := range blks {
|
||||
b.set[b.keyOf(blk.Cid())] = blk
|
||||
b.set[blk.Cid()] = blk
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
|
||||
func (b *mockStore) DeleteBlock(cid cid.Cid) error {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
|
||||
delete(b.set, b.keyOf(cid))
|
||||
delete(b.set, cid)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
|
||||
func (b *mockStore) DeleteMany(cids []cid.Cid) error {
|
||||
b.mx.Lock()
|
||||
defer b.mx.Unlock()
|
||||
|
||||
for _, c := range cids {
|
||||
delete(b.set, b.keyOf(c))
|
||||
delete(b.set, c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *mockStore) Flush(context.Context) error { return nil }
|
||||
|
||||
func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
@@ -768,7 +378,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
|
||||
defer b.mx.Unlock()
|
||||
|
||||
for c := range b.set {
|
||||
err := f(b.cidOf(c))
|
||||
err := f(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,10 +3,11 @@ package splitstore
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
cid "github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
)
|
||||
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
)
|
||||
@@ -21,14 +20,14 @@ var (
|
||||
WarmupBoundary = build.Finality
|
||||
)
|
||||
|
||||
// warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore;
|
||||
// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
|
||||
// this is necessary when we sync from a snapshot or when we enable the splitstore
|
||||
// on top of an existing blockstore (which becomes the coldstore).
|
||||
func (s *SplitStore) warmup(curTs *types.TipSet) error {
|
||||
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
|
||||
return xerrors.Errorf("error locking compaction")
|
||||
}
|
||||
s.compactType = warmup
|
||||
|
||||
go func() {
|
||||
defer atomic.StoreInt32(&s.compacting, 0)
|
||||
|
||||
@@ -56,28 +55,19 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
|
||||
if WarmupBoundary < epoch {
|
||||
boundaryEpoch = epoch - WarmupBoundary
|
||||
}
|
||||
var mx sync.Mutex
|
||||
batchHot := make([]blocks.Block, 0, batchSize)
|
||||
count := new(int64)
|
||||
xcount := new(int64)
|
||||
missing := new(int64)
|
||||
|
||||
visitor, err := s.markSetEnv.New("warmup", 0)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error creating visitor: %w", err)
|
||||
}
|
||||
defer visitor.Close() //nolint
|
||||
|
||||
err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
|
||||
visitor,
|
||||
count := int64(0)
|
||||
xcount := int64(0)
|
||||
missing := int64(0)
|
||||
err := s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup
|
||||
func(c cid.Cid) error {
|
||||
if isUnitaryObject(c) {
|
||||
return errStopWalk
|
||||
}
|
||||
|
||||
atomic.AddInt64(count, 1)
|
||||
count++
|
||||
|
||||
has, err := s.hot.Has(s.ctx, c)
|
||||
has, err := s.hot.Has(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -86,61 +76,59 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
blk, err := s.cold.Get(s.ctx, c)
|
||||
blk, err := s.cold.Get(c)
|
||||
if err != nil {
|
||||
if ipld.IsNotFound(err) {
|
||||
atomic.AddInt64(missing, 1)
|
||||
if err == bstore.ErrNotFound {
|
||||
missing++
|
||||
return errStopWalk
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
atomic.AddInt64(xcount, 1)
|
||||
xcount++
|
||||
|
||||
mx.Lock()
|
||||
batchHot = append(batchHot, blk)
|
||||
if len(batchHot) == batchSize {
|
||||
err = s.hot.PutMany(s.ctx, batchHot)
|
||||
err = s.hot.PutMany(batchHot)
|
||||
if err != nil {
|
||||
mx.Unlock()
|
||||
return err
|
||||
}
|
||||
batchHot = batchHot[:0]
|
||||
}
|
||||
mx.Unlock()
|
||||
|
||||
return nil
|
||||
}, func(cid.Cid) error { return nil })
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(batchHot) > 0 {
|
||||
err = s.hot.PutMany(s.ctx, batchHot)
|
||||
err = s.hot.PutMany(batchHot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)
|
||||
log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
|
||||
|
||||
s.markSetSize = *count + *count>>2 // overestimate a bit
|
||||
err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
|
||||
s.markSetSize = count + count>>2 // overestimate a bit
|
||||
err = s.ds.Put(markSetSizeKey, int64ToBytes(s.markSetSize))
|
||||
if err != nil {
|
||||
log.Warnf("error saving mark set size: %s", err)
|
||||
}
|
||||
|
||||
// save the warmup epoch
|
||||
err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch))
|
||||
err = s.ds.Put(warmupEpochKey, epochToBytes(epoch))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error saving warm up epoch: %w", err)
|
||||
}
|
||||
|
||||
s.warmupEpoch.Store(int64(epoch))
|
||||
s.mx.Lock()
|
||||
s.warmupEpoch = epoch
|
||||
s.mx.Unlock()
|
||||
|
||||
// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
|
||||
err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex))
|
||||
err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("error saving compaction index: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
package splitstore
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// ObjectVisitor is an interface for deduplicating objects during walks
|
||||
type ObjectVisitor interface {
|
||||
Visit(cid.Cid) (bool, error)
|
||||
}
|
||||
|
||||
type noopVisitor struct{}
|
||||
|
||||
var _ ObjectVisitor = (*noopVisitor)(nil)
|
||||
|
||||
func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type tmpVisitor struct {
|
||||
set *cid.Set
|
||||
}
|
||||
|
||||
var _ ObjectVisitor = (*tmpVisitor)(nil)
|
||||
|
||||
func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
|
||||
if isUnitaryObject(c) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return v.set.Visit(c), nil
|
||||
}
|
||||
|
||||
func newTmpVisitor() ObjectVisitor {
|
||||
return &tmpVisitor{set: cid.NewSet()}
|
||||
}
|
||||
|
||||
type concurrentVisitor struct {
|
||||
mx sync.Mutex
|
||||
set *cid.Set
|
||||
}
|
||||
|
||||
var _ ObjectVisitor = (*concurrentVisitor)(nil)
|
||||
|
||||
func newConcurrentVisitor() *concurrentVisitor {
|
||||
return &concurrentVisitor{set: cid.NewSet()}
|
||||
}
|
||||
|
||||
func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
|
||||
if isUnitaryObject(c) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
v.mx.Lock()
|
||||
defer v.mx.Unlock()
|
||||
|
||||
return v.set.Visit(c), nil
|
||||
}
|
||||
+16
-18
@@ -20,55 +20,53 @@ type SyncBlockstore struct {
|
||||
bs MemBlockstore // specifically use a memStore to save indirection overhead.
|
||||
}
|
||||
|
||||
func (*SyncBlockstore) Flush(context.Context) error { return nil }
|
||||
|
||||
func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
|
||||
func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.DeleteBlock(ctx, k)
|
||||
return m.bs.DeleteBlock(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
|
||||
func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.DeleteMany(ctx, ks)
|
||||
return m.bs.DeleteMany(ks)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
|
||||
func (m *SyncBlockstore) Has(k cid.Cid) (bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.Has(ctx, k)
|
||||
return m.bs.Has(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
|
||||
func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
return m.bs.View(ctx, k, callback)
|
||||
return m.bs.View(k, callback)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
|
||||
func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.Get(ctx, k)
|
||||
return m.bs.Get(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
|
||||
func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.bs.GetSize(ctx, k)
|
||||
return m.bs.GetSize(k)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) Put(ctx context.Context, b blocks.Block) error {
|
||||
func (m *SyncBlockstore) Put(b blocks.Block) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.Put(ctx, b)
|
||||
return m.bs.Put(b)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
|
||||
func (m *SyncBlockstore) PutMany(bs []blocks.Block) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.bs.PutMany(ctx, bs)
|
||||
return m.bs.PutMany(bs)
|
||||
}
|
||||
|
||||
func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user