Compare commits

..
Author SHA1 Message Date
Jennifer Wang b2f3825cfa update ffi to include the multi group sdr change 2022-05-04 16:39:31 -04:00
1400 changed files with 60766 additions and 103864 deletions
+653 -934
View File
File diff suppressed because it is too large Load Diff
-2
View File
@@ -106,12 +106,10 @@ func main() {
// form the input data.
type data struct {
Networks []string
ItestFiles []string
UnitSuites map[string]string
}
in := data{
Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"},
ItestFiles: itests,
UnitSuites: func() map[string]string {
ret := make(map[string]string)
+629 -476
View File
File diff suppressed because it is too large Load Diff
+73
View File
@@ -0,0 +1,73 @@
ignore:
# Auto generated
- "^.*_gen.go$"
- "^.*/mock_full.go$"
# Old actors.
- "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
# Tests
- "api/test/**"
- "conformance/**"
# Generators
- "gen/**"
- "chain/actors/agen/**"
# Non-critical utilities
- "api/docgen/**"
- "api/docgen-openrpc/**"
coverage:
status:
patch: off
project:
tools-and-tests:
target: auto
threshold: 1%
informational: true
paths:
- "testplans"
- "tools"
- "system"
- "snap"
- "lotuspond"
- "conformance"
- "scripts"
- "gen"
- "build"
markets:
target: auto
threshold: 1%
informational: false
paths:
- "markets"
- "paychmgr"
miner:
target: auto
threshold: 1.5%
informational: false
paths:
- "miner"
- "storage"
chain:
target: auto
threshold: 1%
informational: false
paths:
- "chain"
node:
target: auto
threshold: 1%
informational: false
paths:
- "node"
- "blockstore"
- "metrics"
- "lib"
- "genesis"
- "gateway"
- "api"
- "journal"
cli:
target: auto
threshold: 1%
informational: true
paths:
- "cli"
- "cmd"
-2
View File
@@ -1,2 +0,0 @@
chain/actors/builtin/*/v* linguist-generated=true
chain/actors/builtin/*/message* linguist-generated=true
+36 -21
View File
@@ -9,9 +9,15 @@ body:
options:
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
required: true
- label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
- label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
required: true
- label: I did not make any code changes to lotus.
required: false
@@ -22,11 +28,19 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
- label: lotus miner - mining and block production
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
required: false
- label: lotus JSON-RPC API
required: false
@@ -42,33 +56,22 @@ body:
description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
placeholder: |
e.g.
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0
Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
validations:
required: true
- type: textarea
id: ReproSteps
attributes:
label: Repro Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doing when you experienced the bug?
* What you were doding when you experienced the bug?
* Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour?
* For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
* For proving issues, include the output of `lotus-miner proving` info.
* For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
validations:
required: true
- type: textarea
@@ -78,8 +81,20 @@ body:
render: text
description: |
Please provide debug logs of the problem, remember you can get set log level control for:
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control).
* lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
* lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem.
If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
validations:
required: true
required: true
- type: textarea
id: RepoSteps
attributes:
label: Repo Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
-8
View File
@@ -1,8 +0,0 @@
blank_issues_enabled: true
contact_links:
- name: Ask a question about Lotus or get support
url: https://github.com/filecoin-project/lotus/discussions/new/choose
about: Ask a question or request support for using Lotus
- name: Filecoin protocol feature or enhancement
url: https://github.com/filecoin-project/FIPs/discussions/new/choose
about: Write a discussion in the Filecoin Improvement Proposal repo
+20 -14
View File
@@ -7,7 +7,13 @@ body:
label: Checklist
description: Please check off the following boxes before continuing to create an improvement suggestion!
options:
- label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature.
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated improvement to propose.
required: true
- type: checkboxes
attributes:
@@ -16,11 +22,19 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
- label: lotus miner - mining and block production
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
required: false
- label: lotus JSON-RPC API
required: false
@@ -31,17 +45,9 @@ body:
- type: textarea
id: request
attributes:
label: Enhancement Suggestion
description: A clear and concise description of the suggested enhancement?
placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to...
validations:
required: true
- type: textarea
id: request
attributes:
label: Use-Case
description: How would this enhancement help you?
placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable...
label: Improvement Suggestion
description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
validations:
required: true
+14 -3
View File
@@ -7,6 +7,8 @@ body:
label: Checklist
description: Please check off the following boxes before continuing to create a new feature request!
options:
- label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is, please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
required: true
- label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
required: true
- label: I **have** a specific, actionable, and well motivated feature request to propose.
@@ -18,11 +20,19 @@ body:
options:
- label: lotus daemon - chain sync
required: false
- label: lotus fvm/fevm - Lotus FVM and FEVM interactions
- label: lotus miner - mining and block production
required: false
- label: lotus miner/worker - sealing
required: false
- label: lotus miner - proving(WindowPoSt/WinningPoSt)
- label: lotus miner - proving(WindowPoSt)
required: false
- label: lotus miner/market - storage deal
required: false
- label: lotus miner/market - retrieval deal
required: false
- label: lotus miner/market - data transfer
required: false
- label: lotus client
required: false
- label: lotus JSON-RPC API
required: false
@@ -46,7 +56,7 @@ body:
validations:
required: true
- type: textarea
id: alternatives
id: alternates
attributes:
label: Describe alternatives you've considered
description: A clear and concise description of any alternative solutions or features you've considered.
@@ -59,3 +69,4 @@ body:
description: Add any other context, design docs or screenshots about the feature request here.
validations:
required: false
@@ -1,83 +0,0 @@
name: "Bug Report - developer/service provider"
description: "Bug report template about FEVM/FVM for developers/service providers"
labels: [need/triage, kind/bug, area/fevm]
body:
- type: checkboxes
attributes:
label: Checklist
description: Please check off the following boxes before continuing to file a bug report!
options:
- label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
required: true
- label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
required: true
- label: I did not make any code changes to lotus.
required: false
- type: checkboxes
attributes:
label: Lotus component
description: Please select the lotus component you are filing a bug for
options:
- label: lotus Ethereum RPC
required: false
- label: lotus FVM - Lotus FVM interactions
required: false
- label: FEVM tooling
required: false
- label: Other
required: false
- type: textarea
id: version
attributes:
label: Lotus Version
render: text
description: Enter the output of `lotus version` if applicable.
placeholder: |
e.g.
Daemon: 1.19.0+mainnet+git.64059ca87+api1.5.0
Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
validations:
required: true
- type: textarea
id: repro
attributes:
label: Repro Steps
description: "Steps to reproduce the behavior"
value: |
1. Run '...'
2. Do '...'
3. See error '...'
...
validations:
required: false
- type: textarea
id: Description
attributes:
label: Describe the Bug
description: |
This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
* What you were doing when you experienced the bug? What are you trying to build?
* Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
* What is the expected behaviour? Links to the actual code?
validations:
required: true
- type: textarea
id: toolingInfo
attributes:
label: Tooling
render: text
description: |
What kind of tooling are you using:
* Are you using ether.js, Alchemy, Hardhat, etc.
validations:
required: true
- type: textarea
id: extraInfo
attributes:
label: Configuration Options
render: text
description: |
Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM
* lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section
validations:
required: true
+11 -13
View File
@@ -1,23 +1,21 @@
## Related Issues
<!-- Link issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made -->
<!-- link all issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made.-->
## Proposed Changes
<!-- A clear list of the changes being made -->
<!-- provide a clear list of the changes being made-->
## Additional Info
<!-- Callouts, links to documentation, and etc -->
<!-- callouts, links to documentation, and etc-->
## Checklist
Before you mark the PR ready for review, please make sure that:
- [ ] Commits have a clear commit message.
- [ ] PR title is in the form of of `<PR type>: <area>: <change being made>`
- example: ` fix: mempool: Introduce a cache for valid signatures`
- `PR type`: fix, feat, build, chore, ci, docs, perf, refactor, revert, style, test
- `area`, e.g. api, chain, state, market, mempool, multisig, networking, paych, proving, sealing, wallet, deps
- [ ] New features have usage guidelines and / or documentation updates in
- [ ] [Lotus Documentation](https://lotus.filecoin.io)
- [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
- [ ] Tests exist for new functionality or change in behavior
- [ ] All commits have a clear commit message.
- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
- example: ` fix: mempool: Introduce a cache for valid signatures`
- `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
- `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
- [ ] This PR has tests for new functionality or change in behaviour
- [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
- [ ] CI is green
+9 -13
View File
@@ -13,14 +13,10 @@ name: "CodeQL"
on:
push:
branches:
- master
- 'release/*'
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches:
- master
- 'release/*'
branches: [ master ]
jobs:
analyze:
@@ -37,17 +33,17 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v2
- uses: actions/setup-go@v3
- uses: actions/setup-go@v1
with:
go-version: '1.18.8'
go-version: '1.17.9'
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v1
with:
languages: go
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
@@ -56,7 +52,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v1
# ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -70,4 +66,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v1
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: testground run
uses: testground/testground-github-action@v1
uses: coryschwartz/testground-github-action@v1.1
with:
backend_addr: ${{ matrix.backend_addr }}
backend_proto: ${{ matrix.backend_proto }}
+7 -10
View File
@@ -1,3 +1,6 @@
/AppDir
/appimage-builder-cache
*.AppImage
/lotus
/lotus-miner
/lotus-worker
@@ -6,6 +9,7 @@
/lotus-chainwatch
/lotus-shed
/lotus-sim
/lotus-pond
/lotus-townhall
/lotus-fountain
/lotus-stats
@@ -17,6 +21,8 @@
/docgen-md
/docgen-openrpc
/bench.json
/lotuspond/front/node_modules
/lotuspond/front/build
/cmd/lotus-townhall/townhall/node_modules
/cmd/lotus-townhall/townhall/build
/cmd/lotus-townhall/townhall/package-lock.json
@@ -35,6 +41,7 @@ build/paramfetch.sh
/darwin
/linux
*.snap
build/builtin-actors
*-fuzz.zip
/chain/types/work_msg/
@@ -42,13 +49,3 @@ bin/ipget
bin/tmp/*
.idea
scratchpad
build/builtin-actors/v*
build/builtin-actors/*.car
dist/
# The following files are checked into git and result
# in dirty git state if removed from the docker context
!extern/filecoin-ffi/rust/filecoin.pc
+3
View File
@@ -43,6 +43,9 @@ issues:
exclude-use-default: false
exclude-rules:
- path: lotuspond
linters:
- errcheck
- path: node/modules/lp2p
linters:
-109
View File
@@ -1,109 +0,0 @@
project_name: lotus
universal_binaries:
- id: lotus
replace: true
name_template: lotus
- id: lotus-miner
replace: true
name_template: lotus-miner
- id: lotus-worker
replace: true
name_template: lotus-worker
builds:
- id: lotus
binary: lotus
builder: prebuilt
goos:
- darwin
- linux
goarch:
- amd64
- arm64
goamd64:
- v1
ignore:
- goos: linux
goarch: arm64
prebuilt:
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus
- id: lotus-miner
binary: lotus-miner
builder: prebuilt
goos:
- darwin
- linux
goarch:
- amd64
- arm64
goamd64:
- v1
ignore:
- goos: linux
goarch: arm64
prebuilt:
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner
- id: lotus-worker
binary: lotus-worker
builder: prebuilt
goos:
- darwin
- linux
goarch:
- amd64
- arm64
goamd64:
- v1
ignore:
- goos: linux
goarch: arm64
prebuilt:
path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker
archives:
- id: primary
format: tar.gz
wrap_in_directory: true
name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}"
files:
# this is a dumb but required hack so it doesn't include the default files
# https://github.com/goreleaser/goreleaser/issues/602
- _n_o_n_e_*
release:
github:
owner: filecoin-project
name: lotus
prerelease: auto
name_template: "v{{.Version}}"
brews:
- tap:
owner: filecoin-project
name: homebrew-lotus
branch: master
ids:
- primary
install: |
bin.install "lotus"
bin.install "lotus-miner"
bin.install "lotus-worker"
test: |
system "#{bin}/lotus --version"
system "#{bin}/lotus-miner --version"
system "#{bin}/lotus-worker --version"
folder: Formula
homepage: "https://filecoin.io"
description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
license: MIT
skip_upload: auto
dependencies:
- name: hwloc
# produced manually so we can include cid checksums
checksum:
disable: true
snapshot:
name_template: "{{ .Version }}"
+1
View File
@@ -0,0 +1 @@
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0" y="0" viewBox="0 0 127 127" xml:space="preserve" enable-background="new 0 0 127 127"><style type="text/css">.st0{fill:#00d2d6}.st1{fill:#fff}</style><g><path class="st0" d="M63.5,127C28.5,127.1-0.2,98.4,0,63.2C0.2,28.3,28.6-0.2,63.9,0c34.8,0.2,63.3,28.7,63.1,64 C126.7,98.7,98.5,127.1,63.5,127z M71.4,57.6c5.5,0.8,11,1.5,16.5,2.3c0.5-1.7,0.9-3.1,1.3-4.7c-5.7-0.8-11.2-1.7-17.1-2.5 c2-7,3.7-13.7,5.8-20.2c0.7-2.2,2.3-4.2,3.9-5.9c2.1-2.2,5-1.7,6.8,0.7c0.7,1,1.4,2.1,2.3,2.9c1.1,1.1,2.8,1.6,4,0.6 c0.8-0.7,0.7-2.4,0.8-3.6c0-0.5-0.6-1.1-1-1.6c-2-2.3-4.7-3.1-7.5-3.2c-6.3-0.3-10.9,3-14.5,7.8c-3.5,4.8-5.1,10.5-6.8,16.2 c-0.5,1.6-0.9,3.3-1.4,5.1c-6.2-0.9-12.1-1.7-18.2-2.6c-0.2,1.6-0.4,3.2-0.6,4.8c6,0.9,11.8,1.8,17.8,2.7c-0.8,3.4-1.5,6.5-2.3,9.7 c-5.8-0.8-11.4-1.6-17-2.4c-0.2,1.8-0.4,3.2-0.6,4.8c5.6,0.9,11,1.7,16.5,2.5c0,0.6,0.1,1,0,1.3c-1.7,7.4-3.4,14.8-5.3,22.2 c-0.9,3.5-2.4,6.9-5.3,9.3c-2.4,2-5,1.7-6.8-0.8c-0.8-1.1-1.5-2.5-2.6-3.3c-0.8-0.6-2.5-0.9-3.1-0.5c-0.9,0.7-1.5,2.2-1.4,3.3 c0.1,1,1,2.2,1.9,2.8c3,2.3,6.5,2.6,10,1.9c5.9-1.2,10.1-4.9,12.7-10.1c2-4.1,3.6-8.5,5-12.9c1.3-4,2.2-8,3.3-12.2 c5.8,0.8,11.5,1.7,17.3,2.5c0.5-1.7,0.9-3.2,1.4-4.8c-6.1-0.9-11.9-1.7-17.7-2.6C70.1,64,70.7,60.9,71.4,57.6z"/><path class="st1" d="M71.4,57.6c-0.7,3.3-1.3,6.4-2,9.8c5.9,0.9,11.7,1.7,17.7,2.6c-0.5,1.6-0.9,3.1-1.4,4.8 c-5.8-0.8-11.5-1.7-17.3-2.5c-1.1,4.2-2,8.3-3.3,12.2c-1.4,4.4-3,8.7-5,12.9c-2.6,5.2-6.8,8.9-12.7,10.1c-3.5,0.7-7,0.4-10-1.9 c-0.9-0.7-1.8-1.8-1.9-2.8c-0.1-1.1,0.5-2.7,1.4-3.3c0.6-0.5,2.3-0.1,3.1,0.5c1.1,0.8,1.8,2.1,2.6,3.3c1.8,2.5,4.4,2.9,6.8,0.8 c2.9-2.5,4.4-5.8,5.3-9.3c1.9-7.3,3.6-14.8,5.3-22.2c0.1-0.3,0-0.7,0-1.3c-5.4-0.8-10.8-1.7-16.5-2.5c0.2-1.6,0.4-3,0.6-4.8 c5.6,0.8,11.1,1.6,17,2.4c0.8-3.2,1.5-6.4,2.3-9.7c-6-0.9-11.7-1.8-17.8-2.7c0.2-1.6,0.4-3.2,0.6-4.8c6.1,0.9,12,1.7,18.2,2.6 c0.5-1.8,0.9-3.5,1.4-5.1c1.7-5.6,3.2-11.3,6.8-16.2c3.6-4.9,8.1-8.1,14.5-7.8c2.8,0.1,5.5,0.9,7.5,3.2c0.4,0.5,1,1.1,1,1.6 c-0.1,1.2,0,2.9-0.8,3.6c-1.1,1.1-2.8,0.5-4-0.6c-0.9-0.9-1.6-1.9-2.3-2.9c-1.8-2.4-4.7-2.9-6.8-0.7c-1.6,1.7-3.2,3.7-3.9,5.9 C75.7,39.4,74,46,72,53c5.9,0.9,11.4,1.7,17.1,2.5c-0.5,1.6-0.9,3.1-1.3,4.7C82.3,59.1,76.9,58.3,71.4,57.6z"/></g></svg>

After

Width:  |  Height:  |  Size: 2.2 KiB

+73
View File
@@ -0,0 +1,73 @@
version: 1
AppDir:
path: ./AppDir
app_info:
id: io.filecoin.lotus
name: Lotus
icon: icon
version: latest
exec: usr/bin/lotus
exec_args: $@
apt:
arch: amd64
allow_unauthenticated: true
sources:
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal main restricted
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal universe
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates universe
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal multiverse
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates multiverse
- sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
universe multiverse
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security main restricted
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security universe
- sourceline: deb http://security.ubuntu.com/ubuntu focal-security multiverse
- sourceline: deb https://cli-assets.heroku.com/apt ./
- sourceline: deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu focal main
- sourceline: deb http://ppa.launchpad.net/git-core/ppa/ubuntu focal main
- sourceline: deb http://archive.canonical.com/ubuntu focal partner
include:
- ocl-icd-libopencl1
- libhwloc15
exclude: []
files:
include:
- /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
- /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
- /usr/lib/x86_64-linux-gnu/libm-2.31.so
- /usr/lib/x86_64-linux-gnu/libdl-2.31.so
- /usr/lib/x86_64-linux-gnu/libc-2.31.so
- /usr/lib/x86_64-linux-gnu/libudev.so.1.6.17
exclude:
- usr/share/man
- usr/share/doc/*/README.*
- usr/share/doc/*/changelog.*
- usr/share/doc/*/NEWS.*
- usr/share/doc/*/TODO.*
test:
fedora:
image: appimagecrafters/tests-env:fedora-30
command: ./AppRun
use_host_x: false
debian:
image: appimagecrafters/tests-env:debian-stable
command: ./AppRun
use_host_x: false
arch:
image: appimagecrafters/tests-env:archlinux-latest
command: ./AppRun
use_host_x: false
centos:
image: appimagecrafters/tests-env:centos-7
command: ./AppRun
use_host_x: false
ubuntu:
image: appimagecrafters/tests-env:ubuntu-xenial
command: ./AppRun
use_host_x: false
AppImage:
arch: x86_64
update-information: guess
sign-key: None
+48 -1717
View File
File diff suppressed because it is too large Load Diff
-136
View File
@@ -1,136 +0,0 @@
#####################################
FROM golang:1.19.7-buster AS lotus-builder
MAINTAINER Lotus Development Team
RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
ENV XDG_CACHE_HOME="/tmp"
### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
PATH=/usr/local/cargo/bin:$PATH \
RUST_VERSION=1.63.0
RUN set -eux; \
dpkgArch="$(dpkg --print-architecture)"; \
case "${dpkgArch##*-}" in \
amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
*) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
esac; \
url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
wget "$url"; \
echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
chmod +x rustup-init; \
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
rm rustup-init; \
chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
rustup --version; \
cargo --version; \
rustc --version;
COPY ./ /opt/filecoin
WORKDIR /opt/filecoin
RUN scripts/docker-git-state-check.sh
### make configurable filecoin-ffi build
ARG FFI_BUILD_FROM_SOURCE=0
ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
RUN make clean deps
ARG RUSTFLAGS=""
ARG GOFLAGS=""
RUN make buildall
#####################################
FROM ubuntu:20.04 AS lotus-base
MAINTAINER Lotus Development Team
# Base resources
COPY --from=lotus-builder /etc/ssl/certs /etc/ssl/certs
COPY --from=lotus-builder /lib/*/libdl.so.2 /lib/
COPY --from=lotus-builder /lib/*/librt.so.1 /lib/
COPY --from=lotus-builder /lib/*/libgcc_s.so.1 /lib/
COPY --from=lotus-builder /lib/*/libutil.so.1 /lib/
COPY --from=lotus-builder /usr/lib/*/libltdl.so.7 /lib/
COPY --from=lotus-builder /usr/lib/*/libnuma.so.1 /lib/
COPY --from=lotus-builder /usr/lib/*/libhwloc.so.5 /lib/
COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
RUN useradd -r -u 532 -U fc \
&& mkdir -p /etc/OpenCL/vendors \
&& echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
#####################################
FROM lotus-base AS lotus
MAINTAINER Lotus Development Team
COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
COPY scripts/docker-lotus-entrypoint.sh /
ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV LOTUS_PATH /var/lib/lotus
ENV DOCKER_LOTUS_IMPORT_WALLET ""
RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus
VOLUME /var/tmp/filecoin-proof-parameters
USER fc
EXPOSE 1234
ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
CMD ["-help"]
#####################################
FROM lotus-base AS lotus-all-in-one
ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
ENV LOTUS_MINER_PATH /var/lib/lotus-miner
ENV LOTUS_PATH /var/lib/lotus
ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
ENV WALLET_PATH /var/lib/lotus-wallet
COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-seed /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/
COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
RUN mkdir /var/tmp/filecoin-proof-parameters
RUN mkdir /var/lib/lotus
RUN mkdir /var/lib/lotus-miner
RUN mkdir /var/lib/lotus-worker
RUN mkdir /var/lib/lotus-wallet
RUN chown fc: /var/tmp/filecoin-proof-parameters
RUN chown fc: /var/lib/lotus
RUN chown fc: /var/lib/lotus-miner
RUN chown fc: /var/lib/lotus-worker
RUN chown fc: /var/lib/lotus-wallet
VOLUME /var/tmp/filecoin-proof-parameters
VOLUME /var/lib/lotus
VOLUME /var/lib/lotus-miner
VOLUME /var/lib/lotus-worker
VOLUME /var/lib/lotus-wallet
EXPOSE 1234
EXPOSE 2345
EXPOSE 3456
EXPOSE 1777
+14 -31
View File
@@ -1,47 +1,30 @@
##### DEPRECATED
FROM golang:1.19.7-buster AS builder-deps
FROM golang:1.17.9-buster AS builder-deps
MAINTAINER Lotus Development Team
RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
ARG RUST_VERSION=nightly
ENV XDG_CACHE_HOME="/tmp"
### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
PATH=/usr/local/cargo/bin:$PATH \
RUST_VERSION=1.63.0
PATH=/usr/local/cargo/bin:$PATH
RUN set -eux; \
dpkgArch="$(dpkg --print-architecture)"; \
case "${dpkgArch##*-}" in \
amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
*) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
esac; \
url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
wget "$url"; \
echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
chmod +x rustup-init; \
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
rm rustup-init; \
chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
rustup --version; \
cargo --version; \
rustc --version;
### end rust
FROM builder-deps AS builder-local
MAINTAINER Lotus Development Team
COPY ./ /opt/filecoin
WORKDIR /opt/filecoin
### make configurable filecoin-ffi build
ARG FFI_BUILD_FROM_SOURCE=0
ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
RUN make clean deps
@@ -69,14 +52,14 @@ MAINTAINER Lotus Development Team
# Base resources
COPY --from=builder /etc/ssl/certs /etc/ssl/certs
COPY --from=builder /lib/*/libdl.so.2 /lib/
COPY --from=builder /lib/*/librt.so.1 /lib/
COPY --from=builder /lib/*/libgcc_s.so.1 /lib/
COPY --from=builder /lib/*/libutil.so.1 /lib/
COPY --from=builder /usr/lib/*/libltdl.so.7 /lib/
COPY --from=builder /usr/lib/*/libnuma.so.1 /lib/
COPY --from=builder /usr/lib/*/libhwloc.so.5 /lib/
COPY --from=builder /usr/lib/*/libOpenCL.so.1 /lib/
COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/
COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1 /lib/
COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/
COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/
COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
RUN useradd -r -u 532 -U fc \
&& mkdir -p /etc/OpenCL/vendors \
-1
View File
@@ -1 +0,0 @@
1.19.7
+28 -41
View File
@@ -8,11 +8,9 @@ unexport GOFLAGS
GOCC?=go
GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
GOVERSIONMIN:=$(shell cat GO_VERSION_MIN | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
ifeq ($(shell expr $(GOVERSION) \< $(GOVERSIONMIN)), 1)
ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
$(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
$(error Update Golang to version to at least $(shell cat GO_VERSION_MIN))
$(error Update Golang to version to at least 1.17.9)
endif
# git modules that need to be loaded
@@ -66,7 +64,7 @@ CLEAN+=build/.update-modules
deps: $(BUILD_DEPS)
.PHONY: deps
build-devnets: build lotus-seed lotus-shed
build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
.PHONY: build-devnets
debug: GOFLAGS+=-tags=debug
@@ -135,18 +133,6 @@ install-worker:
install-app:
install -C ./$(APP) /usr/local/bin/$(APP)
uninstall: uninstall-daemon uninstall-miner uninstall-worker
.PHONY: uninstall
uninstall-daemon:
rm -f /usr/local/bin/lotus
uninstall-miner:
rm -f /usr/local/bin/lotus-miner
uninstall-worker:
rm -f /usr/local/bin/lotus-worker
# TOOLS
lotus-seed: $(BUILD_DEPS)
@@ -162,16 +148,28 @@ benchmarks:
@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
.PHONY: benchmarks
lotus-pond: 2k
$(GOCC) build -o lotus-pond ./lotuspond
.PHONY: lotus-pond
BINS+=lotus-pond
lotus-pond-front:
(cd lotuspond/front && npm i && CI=false npm run build)
.PHONY: lotus-pond-front
lotus-pond-app: lotus-pond-front lotus-pond
.PHONY: lotus-pond-app
lotus-fountain:
rm -f lotus-fountain
$(GOCC) build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
$(GOCC) run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
go build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
.PHONY: lotus-fountain
BINS+=lotus-fountain
lotus-bench:
rm -f lotus-bench
$(GOCC) build $(GOFLAGS) -o lotus-bench ./cmd/lotus-bench
$(GOCC) build -o lotus-bench ./cmd/lotus-bench
.PHONY: lotus-bench
BINS+=lotus-bench
@@ -195,7 +193,7 @@ BINS+=lotus-health
lotus-wallet:
rm -f lotus-wallet
$(GOCC) build $(GOFLAGS) -o lotus-wallet ./cmd/lotus-wallet
$(GOCC) build -o lotus-wallet ./cmd/lotus-wallet
.PHONY: lotus-wallet
BINS+=lotus-wallet
@@ -289,20 +287,14 @@ type-gen: api-gen
$(GOCC) generate -x ./...
goimports -w api/
actors-code-gen:
method-gen: api-gen
(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
actors-gen:
$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
$(GOCC) run ./chain/actors/agen
$(GOCC) fmt ./...
actors-gen: actors-code-gen fiximports
.PHONY: actors-gen
bundle-gen:
$(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES)
$(GOCC) fmt ./build/...
.PHONY: bundle-gen
api-gen:
$(GOCC) run ./gen/api
goimports -w api
@@ -321,7 +313,7 @@ appimage: lotus
cp ./lotus AppDir/usr/bin/
appimage-builder
docsgen: docsgen-md docsgen-openrpc fiximports
docsgen: docsgen-md docsgen-openrpc
docsgen-md-bin: api-gen actors-gen
$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
@@ -338,7 +330,7 @@ docsgen-md-storage: docsgen-md-bin
docsgen-md-worker: docsgen-md-bin
./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
docsgen-openrpc-full: docsgen-openrpc-bin
./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
@@ -346,15 +338,10 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
docsgen-openrpc-worker: docsgen-openrpc-bin
./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
docsgen-openrpc-gateway: docsgen-openrpc-bin
./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
fiximports:
./scripts/fiximports
gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen
@@ -366,7 +353,7 @@ snap: lotus lotus-miner lotus-worker
# separate from gen because it needs binaries
docsgen-cli: lotus lotus-miner lotus-worker
python3 ./scripts/generate-lotus-cli.py
python ./scripts/generate-lotus-cli.py
./lotus config default > documentation/en/default-lotus-config.toml
./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
.PHONY: docsgen-cli
@@ -375,4 +362,4 @@ print-%:
@echo $*=$($*)
circleci:
go generate -x ./.circleci
go generate -x ./.circleci
+13 -13
View File
@@ -1,5 +1,5 @@
<p align="center">
<a href="https://lotus.filecoin.io/" title="Filecoin Docs">
<a href="https://docs.filecoin.io/" title="Filecoin Docs">
<img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
</a>
</p>
@@ -10,7 +10,7 @@
<a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
<a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
<a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.18.8-blue.svg" /></a>
<a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
<br>
</p>
@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
## Reporting a Vulnerability
@@ -31,7 +31,7 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
- [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
- [builtin-actors](https://github.com/filecoin-project/builtin-actors)
- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
## Contribute
@@ -64,17 +64,17 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
Fedora:
```
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
```
For other distributions you can find the required dependencies [here.](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms) For instructions specific to macOS, you can find them [here.](https://lotus.filecoin.io/lotus/install/macos/)
For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
#### Go
To build Lotus, you need a working installation of [Go 1.19.7 or higher](https://golang.org/dl/):
To build Lotus, you need a working installation of [Go 1.17.9 or higher](https://golang.org/dl/):
```bash
wget -c https://golang.org/dl/go1.19.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
wget -c https://golang.org/dl/go1.17.9.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
```
**TIP:**
@@ -101,7 +101,7 @@ Note: The default branch `master` is the dev branch where the latest new feature
2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://lotus.filecoin.io/lotus/manage/switch-networks/) before proceeding.
If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
@@ -113,8 +113,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
Currently, the latest code on the _master_ branch corresponds to mainnet.
3. If you are in China, see "[Lotus: tips when running in China](https://lotus.filecoin.io/lotus/configure/nodes-in-china/)".
4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://lotus.filecoin.io/lotus/install/prerequisites/). Note, if you are building the proof binaries from source, [installing rustup](https://lotus.filecoin.io/lotus/install/linux/#rustup) is also needed.
3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
5. Build and install Lotus:
@@ -129,9 +129,9 @@ Note: The default branch `master` is the dev branch where the latest new feature
This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
`lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://lotus.filecoin.io/lotus/configure/defaults/#environment-variables) for information on how to customize the Lotus folder.
`lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
## License
+3 -7
View File
@@ -3,14 +3,13 @@ package api
import (
"context"
"fmt"
"time"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/journal/alerting"
"github.com/google/uuid"
"github.com/filecoin-project/go-jsonrpc/auth"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/journal/alerting"
)
// MODIFYING THE API INTERFACE
@@ -50,9 +49,6 @@ type Common interface {
// trigger graceful shutdown
Shutdown(context.Context) error //perm:admin
// StartTime returns node start time
StartTime(context.Context) (time.Time, error) //perm:read
// Session returns a random UUID of api provider session
Session(context.Context) (uuid.UUID, error) //perm:read
-42
View File
@@ -1,42 +0,0 @@
package api
import (
"errors"
"reflect"
"github.com/filecoin-project/go-jsonrpc"
)
const (
EOutOfGas = iota + jsonrpc.FirstUserCode
EActorNotFound
)
type ErrOutOfGas struct{}
func (e *ErrOutOfGas) Error() string {
return "call ran out of gas"
}
type ErrActorNotFound struct{}
func (e *ErrActorNotFound) Error() string {
return "actor not found"
}
var RPCErrors = jsonrpc.NewErrors()
func ErrorIsIn(err error, errorTypes []error) bool {
for _, etype := range errorTypes {
tmp := reflect.New(reflect.PointerTo(reflect.ValueOf(etype).Elem().Type())).Interface()
if errors.As(err, tmp) {
return true
}
}
return false
}
func init() {
RPCErrors.Register(EOutOfGas, new(*ErrOutOfGas))
RPCErrors.Register(EActorNotFound, new(*ErrActorNotFound))
}
+33 -214
View File
@@ -6,33 +6,27 @@ import (
"fmt"
"time"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/go-state-types/builtin/v9/market"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/actors/builtin"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/actors/builtin/power"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/imports"
)
@@ -43,7 +37,6 @@ import (
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
}
const LookbackNoLimit = abi.ChainEpoch(-1)
@@ -129,9 +122,6 @@ type FullNode interface {
// ChainHasObj checks if a given CID exists in the chain blockstore.
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
// ChainPutObj puts a given object into the block store
ChainPutObj(context.Context, blocks.Block) error //perm:admin
// ChainStatObj returns statistics about the graph referenced by 'obj'.
// If 'base' is also specified, then the returned stat will be a diff
// between the two objects.
@@ -153,7 +143,7 @@ type FullNode interface {
// ChainGetPath returns a set of revert/apply operations needed to get from
// one tipset to another, for example:
// ```
//```
// to
// ^
// from tAA
@@ -162,7 +152,7 @@ type FullNode interface {
// ^---*--^
// ^
// tRR
// ```
//```
// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) //perm:read
@@ -173,24 +163,6 @@ type FullNode interface {
// If oldmsgskip is set, messages from before the requested roots are also not included.
ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
// ChainExportRangeInternal triggers the export of a chain
// CAR-snapshot directly to disk. It is similar to ChainExport,
// except, depending on options, the snapshot can include receipts,
// messages and stateroots for the length between the specified head
// and tail, thus producing "archival-grade" snapshots that include
// all the on-chain data. The header chain is included back to
// genesis and these snapshots can be used to initialize Filecoin
// nodes.
ChainExportRangeInternal(ctx context.Context, head, tail types.TipSetKey, cfg ChainExportConfig) error //perm:admin
// ChainPrune forces compaction on cold store and garbage collects; only supported if you
// are using the splitstore
ChainPrune(ctx context.Context, opts PruneOpts) error //perm:admin
// ChainHotGC does online (badger) GC on the hot store; only supported if you are using
// the splitstore
ChainHotGC(ctx context.Context, opts HotGCOpts) error //perm:admin
// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
// if supported by the underlying implementation.
ChainCheckBlockstore(context.Context) error //perm:admin
@@ -198,8 +170,13 @@ type FullNode interface {
// ChainBlockstoreInfo returns some basic information about the blockstore
ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
// ChainGetEvents returns the events under an event AMT root CID.
ChainGetEvents(context.Context, cid.Cid) ([]types.Event, error) //perm:read
// MethodGroup: Beacon
// The Beacon method group contains methods for interacting with the random beacon (DRAND)
// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
// the entry has not yet been produced, the call will block until the entry
// becomes available
BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
// GasEstimateFeeCap estimates gas fee cap
GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
@@ -407,12 +384,12 @@ type FullNode interface {
ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
// ClientUnimport removes references to the specified file from filestore
// ClientUnimport(path string)
//ClientUnimport(path string)
// ClientListImports lists imported files and their root CIDs
ClientListImports(ctx context.Context) ([]Import, error) //perm:write
// ClientListAsks() []Ask
//ClientListAsks() []Ask
// MethodGroup: State
// The State methods are used to query, inspect, and interact with chain state.
@@ -427,7 +404,7 @@ type FullNode interface {
StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
//
// If a tipset key is provided, and a replacing message is not found on chain,
// If a tipset key is provided, and a replacing message is found on chain,
// the method will return an error saying that the message wasn't found
//
// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -466,7 +443,7 @@ type FullNode interface {
// StateMinerPower returns the power of the indicated miner
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) //perm:read
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
// StateMinerPartitions returns all partitions in the specified deadline
@@ -483,23 +460,18 @@ type FullNode interface {
StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
// StateMinerSectorAllocated checks if a sector number is marked as allocated.
// StateMinerSectorAllocated checks if a sector is allocated
StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
// Returns nil and no error if the sector isn't precommitted.
//
// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
// StateSearchMsg looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
//
// NOTE: If a replacing message is found on chain, this method will return
@@ -545,35 +517,18 @@ type FullNode interface {
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
// StateMarketDeals returns information about every deal in the Storage Market
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
// StateMarketStorageDeal returns information about the indicated deal
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal. Returns nil if
// pending allocation is not found.
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
// StateGetAllocation returns the allocation for a given address and allocation ID.
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
// StateGetAllocations returns the all the allocations for a given client.
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
// StateGetClaim returns the claim for a given address and claim ID.
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
// StateGetClaims returns the all the claims for a given provider.
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
// StateComputeDataCID computes DataCID from a set of on-chain deals
StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
// StateLookupID retrieves the ID address of the given address
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
// StateAccountKey returns the public key address of the given ID address
StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateLookupRobustAddress returns the public key address of the given ID address for non-account addresses (multisig, miners etc)
StateLookupRobustAddress(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateChangedActors returns all the actors whose states change between the two given state CIDs
// TODO: Should this take tipset keys instead?
StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
// StateMinerAllocated returns a bitfield containing all sector numbers marked as allocated in miner state
StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) //perm:read
// StateCompute is a flexible command that applies the given messages on the given tipset.
// The messages are run as though the VM were at the provided height.
//
@@ -615,7 +570,7 @@ type FullNode interface {
// Returns nil if there is no entry in the data cap table for the
// address.
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
// StateVerifiedClientStatus returns the address of the Verified Registry's root key
StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
// can issue. It takes the deal size and verified status as parameters.
@@ -629,24 +584,12 @@ type FullNode interface {
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
// StateNetworkVersion returns the network version at the given tipset
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
// the entry has not yet been produced, the call will block until the entry
// becomes available
StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
// StateGetNetworkParams return current network params
StateGetNetworkParams(ctx context.Context) (*NetworkParams, error) //perm:read
// MethodGroup: Msig
// The Msig methods are used to interact with multisig wallets on the
// filecoin network
@@ -659,14 +602,14 @@ type FullNode interface {
// It takes the following params: <multisig address>, <start epoch>, <end epoch>
MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
// MsigGetPending returns pending transactions for the given multisig
// wallet. Once pending transactions are fully approved, they will no longer
// appear here.
//MsigGetPending returns pending transactions for the given multisig
//wallet. Once pending transactions are fully approved, they will no longer
//appear here.
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) //perm:read
// MsigCreate creates a multisig wallet
// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
// <initial balance>, <sender address of the create msg>, <gas price>
//<initial balance>, <sender address of the create msg>, <gas price>
MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (*MessagePrototype, error) //perm:sign
// MsigPropose proposes a multisig message
@@ -778,102 +721,11 @@ type FullNode interface {
NodeStatus(ctx context.Context, inclChainStatus bool) (NodeStatus, error) //perm:read
// MethodGroup: Eth
// These methods are used for Ethereum-compatible JSON-RPC calls
//
// EthAccounts will always return [] since we don't expect Lotus to manage private keys
EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) //perm:read
// EthAddressToFilecoinAddress converts an EthAddress into an f410 Filecoin Address
EthAddressToFilecoinAddress(ctx context.Context, ethAddress ethtypes.EthAddress) (address.Address, error) //perm:read
// FilecoinAddressToEthAddress converts an f410 or f0 Filecoin Address to an EthAddress
FilecoinAddressToEthAddress(ctx context.Context, filecoinAddress address.Address) (ethtypes.EthAddress, error) //perm:read
// EthBlockNumber returns the height of the latest (heaviest) TipSet
EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
// EthGetBlockTransactionCountByNumber returns the number of messages in the TipSet
EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error) //perm:read
// EthGetBlockTransactionCountByHash returns the number of messages in the TipSet
EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error) //perm:read
EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error) //perm:read
EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error) //perm:read
EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error) //perm:read
EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error) //perm:read
EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error) //perm:read
EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error) //perm:read
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error) //perm:read
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error) //perm:read
EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error) //perm:read
EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) //perm:read
EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error) //perm:read
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error) //perm:read
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error) //perm:read
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error) //perm:read
EthChainId(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
NetVersion(ctx context.Context) (string, error) //perm:read
NetListening(ctx context.Context) (bool, error) //perm:read
EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error) //perm:read
EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) //perm:read
EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error) //perm:read
EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error) //perm:read
EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam string) (ethtypes.EthBytes, error) //perm:read
EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read
// Returns event logs matching given filter spec.
EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error) //perm:read
// Polling method for a filter, returns event logs which occurred since last poll.
// (requires write perm since timestamp of last filter execution will be written)
EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:write
// Returns event logs matching filter with given id.
// (requires write perm since timestamp of last filter execution will be written)
EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:write
// Installs a persistent filter based on given filter spec.
EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error) //perm:write
// Installs a persistent filter to notify when a new block arrives.
EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:write
// Installs a persistent filter to notify when new messages arrive in the message pool.
EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:write
// Uninstalls a filter with given id.
EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error) //perm:write
// Subscribe to different event types using websockets
// eventTypes is one or more of:
// - newHeads: notify when new blocks arrive.
// - pendingTransactions: notify when new messages arrive in the message pool.
// - logs: notify new event logs that match a criteria
// params contains additional parameters used with the log event type
// The client will receive a stream of EthSubscriptionResponse values until EthUnsubscribe is called.
EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error) //perm:write
// Unsubscribe from a websocket subscription
EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error) //perm:write
// Returns the client version
Web3ClientVersion(ctx context.Context) (string, error) //perm:read
// CreateBackup creates node backup onder the specified file name. The
// method requires that the lotus daemon is running with the
// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error //perm:admin
RaftState(ctx context.Context) (*RaftStateData, error) //perm:read
RaftLeader(ctx context.Context) (peer.ID, error) //perm:read
}
// reverse interface to the client, called after EthSubscribe
type EthSubscriber interface {
// note: the parameter is ethtypes.EthSubscriptionResponse serialized as json object
EthSubscription(ctx context.Context, r jsonrpc.RawParams) error // rpc_method:eth_subscription notify:true
}
type StorageAsk struct {
@@ -1123,12 +975,8 @@ type RetrievalOrder struct {
Client address.Address
Miner address.Address
MinerPeer *retrievalmarket.RetrievalPeer
RemoteStore *RemoteStoreID `json:"RemoteStore,omitempty"`
}
type RemoteStoreID = uuid.UUID
type InvocResult struct {
MsgCid cid.Cid
Msg *types.Message
@@ -1354,32 +1202,3 @@ type MsigTransaction struct {
Approved []address.Address
}
type PruneOpts struct {
MovingGC bool
RetainState int64
}
type HotGCOpts struct {
Threshold float64
Periodic bool
Moving bool
}
type EthTxReceipt struct {
TransactionHash ethtypes.EthHash `json:"transactionHash"`
TransactionIndex ethtypes.EthUint64 `json:"transactionIndex"`
BlockHash ethtypes.EthHash `json:"blockHash"`
BlockNumber ethtypes.EthUint64 `json:"blockNumber"`
From ethtypes.EthAddress `json:"from"`
To *ethtypes.EthAddress `json:"to"`
StateRoot ethtypes.EthHash `json:"root"`
Status ethtypes.EthUint64 `json:"status"`
ContractAddress *ethtypes.EthAddress `json:"contractAddress"`
CumulativeGasUsed ethtypes.EthUint64 `json:"cumulativeGasUsed"`
GasUsed ethtypes.EthUint64 `json:"gasUsed"`
EffectiveGasPrice ethtypes.EthBigInt `json:"effectiveGasPrice"`
LogsBloom ethtypes.EthBytes `json:"logsBloom"`
Logs []ethtypes.EthLog `json:"logs"`
Type ethtypes.EthUint64 `json:"type"`
}
+4 -52
View File
@@ -4,18 +4,14 @@ import (
"context"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
"github.com/filecoin-project/go-state-types/dline"
apitypes "github.com/filecoin-project/lotus/api/types"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// MODIFYING THE API INTERFACE
@@ -26,7 +22,7 @@ import (
// When adding / changing methods in this file:
// * Do the change here
// * Adjust implementation in `node/impl/`
// * Run `make clean && make deps && make gen` - this will:
// * Run `make gen` - this will:
// * Generate proxy structs
// * Generate mocks
// * Generate markdown docs
@@ -34,7 +30,6 @@ import (
type Gateway interface {
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
ChainHead(ctx context.Context) (*types.TipSet, error)
ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
@@ -48,70 +43,27 @@ type Gateway interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainGetGenesis(context.Context) (*types.TipSet, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
Version(context.Context) (APIVersion, error)
Discover(context.Context) (apitypes.OpenRPCDocument, error)
EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error)
EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error)
EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error)
EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error)
EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error)
EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error)
EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error)
EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error)
EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error)
EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error)
EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkOpt string) (ethtypes.EthUint64, error)
EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error)
EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error)
EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkOpt string) (ethtypes.EthBytes, error)
EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam string) (ethtypes.EthBytes, error)
EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam string) (ethtypes.EthBigInt, error)
EthChainId(ctx context.Context) (ethtypes.EthUint64, error)
NetVersion(ctx context.Context) (string, error)
NetListening(ctx context.Context) (bool, error)
EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error)
EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam string) (ethtypes.EthBytes, error)
EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)
EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error)
EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error)
EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error)
EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error)
EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error)
EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error)
Web3ClientVersion(ctx context.Context) (string, error)
}
+4 -4
View File
@@ -4,10 +4,10 @@ import (
"context"
"time"
"github.com/libp2p/go-libp2p/core/metrics"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/protocol"
metrics "github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/protocol"
)
// MODIFYING THE API INTERFACE
+51 -230
View File
@@ -5,28 +5,28 @@ import (
"context"
"time"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/abi"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/go-fil-markets/piecestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin/v9/market"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
abinetwork "github.com/filecoin-project/go-state-types/network"
builtinactors "github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
)
// MODIFYING THE API INTERFACE
@@ -50,22 +50,10 @@ type StorageMiner interface {
ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
ActorAddressConfig(ctx context.Context) (AddressConfig, error) //perm:read
// WithdrawBalance allows to withdraw balance from miner actor to owner address
// Specify amount as "0" to withdraw full balance. This method returns a message CID
// and does not wait for message execution
ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
// BeneficiaryWithdrawBalance allows the beneficiary of a miner to withdraw balance from miner actor
// Specify amount as "0" to withdraw full balance. This method returns a message CID
// and does not wait for message execution
BeneficiaryWithdrawBalance(context.Context, abi.TokenAmount) (cid.Cid, error) //perm:admin
MiningBase(context.Context) (*types.TipSet, error) //perm:read
ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
// Temp api for testing
PledgeSector(context.Context) (abi.SectorID, error) //perm:write
@@ -75,9 +63,9 @@ type StorageMiner interface {
// Add piece to an open sector. If no sectors with enough space are open,
// either a new sector will be created, or this call will block until more
// sectors can be created.
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
// List all staged sectors
SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@@ -130,93 +118,52 @@ type StorageMiner interface {
// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
// SectorNumAssignerMeta returns sector number assigner metadata - reserved/allocated
SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
// SectorNumReservations returns a list of sector number reservations
SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
// SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
// numbers or name. Set force to true to override safety checks.
// Valid characters for name: a-z, A-Z, 0-9, _, -
SectorNumReserve(ctx context.Context, name string, sectors bitfield.BitField, force bool) error //perm:admin
// SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
// by default lotus will allocate lowest-available sector numbers to the reservation.
// For restrictions on `name` see SectorNumReserve
SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) //perm:admin
// SectorNumFree drops a sector reservation
SectorNumFree(ctx context.Context, name string) error //perm:admin
SectorReceive(ctx context.Context, meta RemoteSectorMeta) error //perm:admin
// WorkerConnect tells the node to connect to workers RPC
WorkerConnect(context.Context, string) error //perm:admin retry:true
WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) //perm:admin
// storiface.WorkerReturn
ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true
ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
ReturnDownloadSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
//storiface.WorkerReturn
ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error //perm:admin retry:true
ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true
ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error //perm:admin retry:true
ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
// SealingSchedDiag dumps internal sealing scheduler state
SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
SealingAbort(ctx context.Context, call storiface.CallID) error //perm:admin
// SealingSchedRemove removes a request from sealing pipeline
SealingRemoveRequest(ctx context.Context, schedId uuid.UUID) error //perm:admin
// paths.SectorIndex
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
StorageDetach(ctx context.Context, id storiface.ID, url string) error //perm:admin
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
// StorageFindSector returns list of paths where the specified sector files exist.
//
// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
// - Paths which have sector files locally (don't require fetching) will be listed first.
// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
// file types are specified, each type will be considered individually, and a union of all paths
// which can accommodate each file type will be returned.
// SectorIndex
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
StorageAddLocal(ctx context.Context, path string) error //perm:admin
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
MarketListDeals(ctx context.Context) ([]*MarketDeal, error) //perm:read
// MarketListRetrievalDeals is deprecated, returns empty list
MarketListRetrievalDeals(ctx context.Context) ([]struct{}, error) //perm:read
MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
MarketListDeals(ctx context.Context) ([]MarketDeal, error) //perm:read
MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error) //perm:read
MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error) //perm:read
MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) //perm:read
MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
@@ -277,9 +224,6 @@ type StorageMiner interface {
// DagstoreGC runs garbage collection on the DAG store.
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
// so they can download its index
IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
@@ -295,7 +239,7 @@ type StorageMiner interface {
RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
DealsList(ctx context.Context) ([]*MarketDeal, error) //perm:admin
DealsList(ctx context.Context) ([]MarketDeal, error) //perm:admin
DealsConsiderOnlineStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderOnlineStorageDeals(context.Context, bool) error //perm:admin
DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error) //perm:admin
@@ -311,6 +255,8 @@ type StorageMiner interface {
DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error) //perm:admin
DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
PiecesListPieces(ctx context.Context) ([]cid.Cid, error) //perm:read
PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) //perm:read
PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
@@ -322,14 +268,9 @@ type StorageMiner interface {
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef) (map[abi.SectorNumber]string, error) //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
ComputeProof(ctx context.Context, ssi []builtinactors.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtinactors.PoStProof, error) //perm:read
// RecoverFault can be used to declare recoveries manually. It sends messages
// to the miner actor with details of recovered sectors and returns the CID of messages. It honors the
// maxPartitionsPerRecoveryMessage from the config
RecoverFault(ctx context.Context, sectors []abi.SectorNumber) ([]cid.Cid, error) //perm:admin
ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
}
var _ storiface.WorkerReturn = *new(StorageMiner)
@@ -419,10 +360,6 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
type SectorState string
func (s *SectorState) String() string {
return string(*s)
}
type AddrUse int
const (
@@ -504,119 +441,3 @@ type DagstoreInitializeAllEvent struct {
Total int
Current int
}
type NumAssignerMeta struct {
Reserved bitfield.BitField
Allocated bitfield.BitField
// ChainAllocated+Reserved+Allocated
InUse bitfield.BitField
Next abi.SectorNumber
}
type RemoteSectorMeta struct {
////////
// BASIC SECTOR INFORMATION
// State specifies the first state the sector will enter after being imported
// Must be one of the following states:
// * Packing
// * GetTicket
// * PreCommitting
// * SubmitCommit
// * Proving/Available
State SectorState
Sector abi.SectorID
Type abi.RegisteredSealProof
////////
// SEALING METADATA
// (allows lotus to continue the sealing process)
// Required in Packing and later
Pieces []SectorPiece // todo better type?
// Required in PreCommitting and later
TicketValue abi.SealRandomness
TicketEpoch abi.ChainEpoch
PreCommit1Out storiface.PreCommit1Out // todo specify better
CommD *cid.Cid
CommR *cid.Cid // SectorKey
// Required in SubmitCommit and later
PreCommitInfo *miner.SectorPreCommitInfo
PreCommitDeposit *big.Int
PreCommitMessage *cid.Cid
PreCommitTipSet types.TipSetKey
SeedValue abi.InteractiveSealRandomness
SeedEpoch abi.ChainEpoch
CommitProof []byte
// Required in Proving/Available
CommitMessage *cid.Cid
// Optional sector metadata to import
Log []SectorLog
////////
// SECTOR DATA SOURCE
// Sector urls - lotus will use those for fetching files into local storage
// Required in all states
DataUnsealed *storiface.SectorLocation
// Required in PreCommitting and later
DataSealed *storiface.SectorLocation
DataCache *storiface.SectorLocation
////////
// SEALING SERVICE HOOKS
// URL
// RemoteCommit1Endpoint is an URL of POST endpoint which lotus will call requesting Commit1 (seal_commit_phase1)
// request body will be json-serialized RemoteCommit1Params struct
RemoteCommit1Endpoint string
// RemoteCommit2Endpoint is an URL of POST endpoint which lotus will call requesting Commit2 (seal_commit_phase2)
// request body will be json-serialized RemoteCommit2Params struct
RemoteCommit2Endpoint string
// RemoteSealingDoneEndpoint is called after the sector exists the sealing pipeline
// request body will be json-serialized RemoteSealingDoneParams struct
RemoteSealingDoneEndpoint string
}
type RemoteCommit1Params struct {
Ticket, Seed []byte
Unsealed cid.Cid
Sealed cid.Cid
ProofType abi.RegisteredSealProof
}
type RemoteCommit2Params struct {
Sector abi.SectorID
ProofType abi.RegisteredSealProof
// todo spec better
Commit1Out storiface.Commit1Out
}
type RemoteSealingDoneParams struct {
// Successful is true if the sector has entered state considered as "successfully sealed"
Successful bool
// State is the state the sector has entered
// For example "Proving" / "Removing"
State string
// Optional commit message CID
CommitMessage *cid.Cid
}
+1 -19
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package api
import (
@@ -12,9 +12,6 @@ import (
"testing"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-jsonrpc"
)
func goCmd() string {
@@ -127,18 +124,3 @@ func TestPermTags(t *testing.T) {
_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
_ = PermissionedWorkerAPI(&WorkerStruct{})
}
func TestRetryErrorIsInTrue(t *testing.T) {
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
require.True(t, ErrorIsIn(&jsonrpc.RPCConnectionError{}, errorsToRetry))
}
func TestRetryErrorIsInFalse(t *testing.T) {
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
require.False(t, ErrorIsIn(xerrors.Errorf("random error"), errorsToRetry))
}
func TestRetryWrappedErrorIsInTrue(t *testing.T) {
errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
require.True(t, ErrorIsIn(xerrors.Errorf("wrapped: %w", &jsonrpc.RPCConnectionError{}), errorsToRetry))
}
+20 -29
View File
@@ -7,10 +7,11 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/specs-storage/storage"
)
// MODIFYING THE API INTERFACE
@@ -33,23 +34,21 @@ type Worker interface {
Info(context.Context) (storiface.WorkerInfo, error) //perm:admin
// storiface.WorkerCalls
DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error) //perm:admin
SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error) //perm:admin
SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error) //perm:admin
FinalizeSector(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) //perm:admin
FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error) //perm:admin
ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin
GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error) //perm:admin
ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error) //perm:admin
MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error) //perm:admin
AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error) //perm:admin
SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error) //perm:admin
SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error) //perm:admin
FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin
GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error) //perm:admin
ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error) //perm:admin
MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin
UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin
Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error) //perm:admin
GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error) //perm:admin
GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
@@ -60,11 +59,7 @@ type Worker interface {
// Storage / Other
Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
StorageDetachLocal(ctx context.Context, path string) error //perm:admin
StorageDetachAll(ctx context.Context) error //perm:admin
StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
StorageAddLocal(ctx context.Context, path string) error //perm:admin
// SetEnabled marks the worker as enabled/disabled. Not that this setting
// may take a few seconds to propagate to task scheduler
@@ -81,10 +76,6 @@ type Worker interface {
// Like ProcessSession, but returns an error when worker is disabled
Session(context.Context) (uuid.UUID, error) //perm:admin
// Trigger shutdown
Shutdown(context.Context) error //perm:admin
}
var _ storiface.WorkerCalls = *new(Worker)
+336 -504
View File
File diff suppressed because it is too large Load Diff
+6 -9
View File
@@ -19,7 +19,7 @@ import (
func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
var res v0api.CommonNetStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
@@ -29,16 +29,16 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
var res v0api.FullNodeStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
// NewFullNodeRPCV1 creates a new http jsonrpc client.
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.FullNode, jsonrpc.ClientCloser, error) {
func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
var res v1api.FullNodeStruct
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res), requestHeader, append([]jsonrpc.Option{jsonrpc.WithErrors(api.RPCErrors)}, opts...)...)
api.GetInternalStructs(&res), requestHeader)
return &res, closer, err
}
@@ -72,7 +72,6 @@ func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.H
api.GetInternalStructs(&res), requestHeader,
append([]jsonrpc.Option{
rpcenc.ReaderParamEncoder(pushUrl),
jsonrpc.WithErrors(api.RPCErrors),
}, opts...)...)
return &res, closer, err
@@ -91,7 +90,6 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
rpcenc.ReaderParamEncoder(pushUrl),
jsonrpc.WithNoReconnect(),
jsonrpc.WithTimeout(30*time.Second),
jsonrpc.WithErrors(api.RPCErrors),
)
return &res, closer, err
@@ -103,7 +101,7 @@ func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res),
requestHeader,
append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
opts...,
)
return &res, closer, err
@@ -115,7 +113,7 @@ func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res),
requestHeader,
append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
opts...,
)
return &res, closer, err
@@ -126,7 +124,6 @@ func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header)
closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
api.GetInternalStructs(&res),
requestHeader,
jsonrpc.WithErrors(api.RPCErrors),
)
return &res, closer, err
+1
View File
@@ -8,6 +8,7 @@ import (
"os"
"github.com/filecoin-project/lotus/api/docgen"
docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
)
+2 -3
View File
@@ -8,11 +8,10 @@ import (
"github.com/alecthomas/jsonschema"
go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
"github.com/ipfs/go-cid"
meta_schema "github.com/open-rpc/meta-schema"
"github.com/filecoin-project/lotus/api/docgen"
"github.com/filecoin-project/lotus/build"
"github.com/ipfs/go-cid"
meta_schema "github.com/open-rpc/meta-schema"
)
// schemaDictEntry represents a type association passed to the jsonschema reflector.
+21 -103
View File
@@ -6,33 +6,31 @@ import (
"go/ast"
"go/parser"
"go/token"
"net/http"
"path/filepath"
"reflect"
"strings"
"time"
"unicode"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
blocks "github.com/ipfs/go-libipfs/blocks"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/protocol"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/metrics"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/protocol"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
"github.com/filecoin-project/go-fil-markets/filestore"
datatransfer "github.com/filecoin-project/go-data-transfer"
filestore "github.com/filecoin-project/go-fil-markets/filestore"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-jsonrpc/auth"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/exitcode"
@@ -41,12 +39,11 @@ import (
"github.com/filecoin-project/lotus/api/v0api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/imports"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
var ExampleValues = map[reflect.Type]interface{}{
@@ -69,7 +66,6 @@ func init() {
}
ExampleValues[reflect.TypeOf(c)] = c
ExampleValues[reflect.TypeOf(&c)] = &c
c2, err := cid.Decode("bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve")
if err != nil {
@@ -99,9 +95,6 @@ func init() {
apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
clientEvent := retrievalmarket.ClientEventDealAccepted
block := blocks.Block(&blocks.BasicBlock{})
ExampleValues[reflect.TypeOf(&block).Elem()] = block
addExample(bitfield.NewFromSet([]uint64{5}))
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
@@ -140,20 +133,12 @@ func init() {
addExample(&textSelExample)
addExample(&apiSelExample)
addExample(network.ReachabilityPublic)
addExample(build.TestNetworkVersion)
allocationId := verifreg.AllocationId(0)
addExample(allocationId)
addExample(&allocationId)
addExample(map[verifreg.AllocationId]verifreg.Allocation{})
claimId := verifreg.ClaimId(0)
addExample(claimId)
addExample(&claimId)
addExample(map[verifreg.ClaimId]verifreg.Claim{})
addExample(build.NewestNetworkVersion)
addExample(map[string]int{"name": 42})
addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
addExample(&types.ExecutionTrace{
Msg: ExampleValue("init", reflect.TypeOf(types.MessageTrace{}), nil).(types.MessageTrace),
MsgRct: ExampleValue("init", reflect.TypeOf(types.ReturnTrace{}), nil).(types.ReturnTrace),
Msg: ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
})
addExample(map[string]types.Actor{
"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
@@ -161,10 +146,6 @@ func init() {
addExample(map[string]api.MarketDeal{
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
})
addExample(map[string]*api.MarketDeal{
"t026363": ExampleValue("init", reflect.TypeOf(&api.MarketDeal{}), nil).(*api.MarketDeal),
})
addExample(map[string]api.MarketBalance{
"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
})
@@ -207,7 +188,6 @@ func init() {
addExample(&si)
addExample(retrievalmarket.DealID(5))
addExample(abi.ActorID(1000))
addExample(map[string]cid.Cid{})
addExample(map[string][]api.SealedRef{
"98000": {
api.SealedRef{
@@ -284,8 +264,6 @@ func init() {
Read: [storiface.FileTypes]uint{2, 3, 0},
},
})
storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
addExample(&storifaceid)
// worker specific
addExample(storiface.AcquireMove)
@@ -300,8 +278,7 @@ func init() {
"title": "Lotus RPC API",
"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
},
"methods": []interface{}{},
},
"methods": []interface{}{}},
)
addExample(api.CheckStatusCode(0))
@@ -338,8 +315,7 @@ func init() {
NumConnsInbound: 3,
NumConnsOutbound: 4,
NumFD: 5,
},
})
}})
addExample(api.NetLimit{
Memory: 123,
StreamsInbound: 1,
@@ -351,63 +327,10 @@ func init() {
FD: 5,
})
addExample(map[string]bitfield.BitField{
"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
})
addExample(&api.RaftStateData{
NonceMap: make(map[address.Address]uint64),
MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
})
addExample(http.Header{
"Authorization": []string{"Bearer ey.."},
})
addExample(map[storiface.SectorFileType]storiface.SectorLocation{
storiface.FTSealed: {
Local: false,
URL: "https://example.com/sealingservice/sectors/s-f0123-12345",
Headers: nil,
},
})
ethint := ethtypes.EthUint64(5)
addExample(ethint)
addExample(&ethint)
ethaddr, _ := ethtypes.ParseEthAddress("0x5CbEeCF99d3fDB3f25E309Cc264f240bb0664031")
addExample(ethaddr)
addExample(&ethaddr)
ethhash, _ := ethtypes.EthHashFromCid(c)
addExample(ethhash)
addExample(&ethhash)
ethFeeHistoryReward := [][]ethtypes.EthBigInt{}
addExample(&ethFeeHistoryReward)
addExample(&uuid.UUID{})
filterid := ethtypes.EthFilterID(ethhash)
addExample(filterid)
addExample(&filterid)
subid := ethtypes.EthSubscriptionID(ethhash)
addExample(subid)
addExample(&subid)
pstring := func(s string) *string { return &s }
addExample(&ethtypes.EthFilterSpec{
FromBlock: pstring("2301220"),
Address: []ethtypes.EthAddress{ethaddr},
})
percent := types.Percent(123)
addExample(percent)
addExample(&percent)
}
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
switch pkg {
case "api": // latest
switch name {
@@ -427,10 +350,6 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
i = &api.WorkerStruct{}
t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
case "Gateway":
i = &api.GatewayStruct{}
t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
default:
panic("unknown type")
}
@@ -477,7 +396,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
case reflect.Ptr:
if t.Elem().Kind() == reflect.Struct {
es := exampleStruct(method, t.Elem(), t)
ExampleValues[t] = es
//ExampleValues[t] = es
return es
}
case reflect.Interface:
@@ -494,8 +413,7 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
if f.Type == parent {
continue
}
if f.IsExported() {
if strings.Title(f.Name) == f.Name {
ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
}
}
-46
View File
@@ -1,46 +0,0 @@
package api
import apitypes "github.com/filecoin-project/lotus/api/types"
func CreateEthRPCAliases(as apitypes.Aliaser) {
// TODO: maybe use reflect to automatically register all the eth aliases
as.AliasMethod("eth_accounts", "Filecoin.EthAccounts")
as.AliasMethod("eth_blockNumber", "Filecoin.EthBlockNumber")
as.AliasMethod("eth_getBlockTransactionCountByNumber", "Filecoin.EthGetBlockTransactionCountByNumber")
as.AliasMethod("eth_getBlockTransactionCountByHash", "Filecoin.EthGetBlockTransactionCountByHash")
as.AliasMethod("eth_getBlockByHash", "Filecoin.EthGetBlockByHash")
as.AliasMethod("eth_getBlockByNumber", "Filecoin.EthGetBlockByNumber")
as.AliasMethod("eth_getTransactionByHash", "Filecoin.EthGetTransactionByHash")
as.AliasMethod("eth_getTransactionCount", "Filecoin.EthGetTransactionCount")
as.AliasMethod("eth_getTransactionReceipt", "Filecoin.EthGetTransactionReceipt")
as.AliasMethod("eth_getTransactionByBlockHashAndIndex", "Filecoin.EthGetTransactionByBlockHashAndIndex")
as.AliasMethod("eth_getTransactionByBlockNumberAndIndex", "Filecoin.EthGetTransactionByBlockNumberAndIndex")
as.AliasMethod("eth_getCode", "Filecoin.EthGetCode")
as.AliasMethod("eth_getStorageAt", "Filecoin.EthGetStorageAt")
as.AliasMethod("eth_getBalance", "Filecoin.EthGetBalance")
as.AliasMethod("eth_chainId", "Filecoin.EthChainId")
as.AliasMethod("eth_feeHistory", "Filecoin.EthFeeHistory")
as.AliasMethod("eth_protocolVersion", "Filecoin.EthProtocolVersion")
as.AliasMethod("eth_maxPriorityFeePerGas", "Filecoin.EthMaxPriorityFeePerGas")
as.AliasMethod("eth_gasPrice", "Filecoin.EthGasPrice")
as.AliasMethod("eth_sendRawTransaction", "Filecoin.EthSendRawTransaction")
as.AliasMethod("eth_estimateGas", "Filecoin.EthEstimateGas")
as.AliasMethod("eth_call", "Filecoin.EthCall")
as.AliasMethod("eth_getLogs", "Filecoin.EthGetLogs")
as.AliasMethod("eth_getFilterChanges", "Filecoin.EthGetFilterChanges")
as.AliasMethod("eth_getFilterLogs", "Filecoin.EthGetFilterLogs")
as.AliasMethod("eth_newFilter", "Filecoin.EthNewFilter")
as.AliasMethod("eth_newBlockFilter", "Filecoin.EthNewBlockFilter")
as.AliasMethod("eth_newPendingTransactionFilter", "Filecoin.EthNewPendingTransactionFilter")
as.AliasMethod("eth_uninstallFilter", "Filecoin.EthUninstallFilter")
as.AliasMethod("eth_subscribe", "Filecoin.EthSubscribe")
as.AliasMethod("eth_unsubscribe", "Filecoin.EthUnsubscribe")
as.AliasMethod("net_version", "Filecoin.NetVersion")
as.AliasMethod("net_listening", "Filecoin.NetListening")
as.AliasMethod("web3_clientVersion", "Filecoin.Web3ClientVersion")
}
+42 -929
View File
File diff suppressed because it is too large Load Diff
+542 -2255
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package api
import (
+30 -151
View File
@@ -5,26 +5,22 @@ import (
"fmt"
"time"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/dagjson"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
ma "github.com/multiformats/go-multiaddr"
"github.com/libp2p/go-libp2p-core/network"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
"github.com/libp2p/go-libp2p-core/peer"
pubsub "github.com/libp2p/go-libp2p-pubsub"
ma "github.com/multiformats/go-multiaddr"
)
// TODO: check if this exists anywhere else
type MultiaddrSlice []ma.Multiaddr
func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
@@ -57,13 +53,7 @@ type PubsubScore struct {
}
type MessageSendSpec struct {
MaxFee abi.TokenAmount
MsgUuid uuid.UUID
}
type MpoolMessageWhole struct {
Msg *types.Message
Spec *MessageSendSpec
MaxFee abi.TokenAmount
}
// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
@@ -112,12 +102,16 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
IsSender: channelState.Sender() == hostID,
Message: channelState.Message(),
}
voucher := channelState.Voucher()
voucherJSON, err := ipld.Encode(voucher.Voucher, dagjson.Encode)
if err != nil {
channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
stringer, ok := channelState.Voucher().(fmt.Stringer)
if ok {
channel.Voucher = stringer.String()
} else {
channel.Voucher = string(voucherJSON)
voucherJSON, err := json.Marshal(channelState.Voucher())
if err != nil {
channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
} else {
channel.Voucher = string(voucherJSON)
}
}
if channel.IsSender {
channel.IsInitiator = !channelState.IsPull()
@@ -140,7 +134,13 @@ type NetStat struct {
}
type NetLimit struct {
Dynamic bool `json:",omitempty"`
// set if Dynamic is false
Memory int64 `json:",omitempty"`
// set if Dynamic is true
MemoryFraction float64 `json:",omitempty"`
MinMemory int64 `json:",omitempty"`
MaxMemory int64 `json:",omitempty"`
Streams, StreamsInbound, StreamsOutbound int
Conns, ConnsInbound, ConnsOutbound int
@@ -252,10 +252,10 @@ type RestrievalRes struct {
}
// Selector specifies ipld selector string
// - if the string starts with '{', it's interpreted as json selector string
// see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
// see https://github.com/ipld/go-ipld-selector-text-lite
// - if the string starts with '{', it's interpreted as json selector string
// see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
// see https://github.com/ipld/go-ipld-selector-text-lite
type Selector string
type DagSpec struct {
@@ -286,124 +286,3 @@ type ExportRef struct {
FromLocalCAR string // if specified, get data from a local CARv2 file.
DealID retrievalmarket.DealID
}
type MinerInfo struct {
Owner address.Address // Must be an ID-address.
Worker address.Address // Must be an ID-address.
NewWorker address.Address // Must be an ID-address.
ControlAddresses []address.Address // Must be an ID-addresses.
WorkerChangeEpoch abi.ChainEpoch
PeerId *peer.ID
Multiaddrs []abi.Multiaddrs
WindowPoStProofType abi.RegisteredPoStProof
SectorSize abi.SectorSize
WindowPoStPartitionSectors uint64
ConsensusFaultElapsed abi.ChainEpoch
Beneficiary address.Address
BeneficiaryTerm *miner.BeneficiaryTerm
PendingBeneficiaryTerm *miner.PendingBeneficiaryChange
}
type NetworkParams struct {
NetworkName dtypes.NetworkName
BlockDelaySecs uint64
ConsensusMinerMinPower abi.StoragePower
SupportedProofTypes []abi.RegisteredSealProof
PreCommitChallengeDelay abi.ChainEpoch
ForkUpgradeParams ForkUpgradeParams
}
type ForkUpgradeParams struct {
UpgradeSmokeHeight abi.ChainEpoch
UpgradeBreezeHeight abi.ChainEpoch
UpgradeIgnitionHeight abi.ChainEpoch
UpgradeLiftoffHeight abi.ChainEpoch
UpgradeAssemblyHeight abi.ChainEpoch
UpgradeRefuelHeight abi.ChainEpoch
UpgradeTapeHeight abi.ChainEpoch
UpgradeKumquatHeight abi.ChainEpoch
UpgradePriceListOopsHeight abi.ChainEpoch
BreezeGasTampingDuration abi.ChainEpoch
UpgradeCalicoHeight abi.ChainEpoch
UpgradePersianHeight abi.ChainEpoch
UpgradeOrangeHeight abi.ChainEpoch
UpgradeClausHeight abi.ChainEpoch
UpgradeTrustHeight abi.ChainEpoch
UpgradeNorwegianHeight abi.ChainEpoch
UpgradeTurboHeight abi.ChainEpoch
UpgradeHyperdriveHeight abi.ChainEpoch
UpgradeChocolateHeight abi.ChainEpoch
UpgradeOhSnapHeight abi.ChainEpoch
UpgradeSkyrHeight abi.ChainEpoch
UpgradeSharkHeight abi.ChainEpoch
UpgradeHyggeHeight abi.ChainEpoch
UpgradeLightningHeight abi.ChainEpoch
UpgradeThunderHeight abi.ChainEpoch
}
type NonceMapType map[address.Address]uint64
type MsgUuidMapType map[uuid.UUID]*types.SignedMessage
type RaftStateData struct {
NonceMap NonceMapType
MsgUuids MsgUuidMapType
}
func (n *NonceMapType) MarshalJSON() ([]byte, error) {
marshalled := make(map[string]uint64)
for a, n := range *n {
marshalled[a.String()] = n
}
return json.Marshal(marshalled)
}
func (n *NonceMapType) UnmarshalJSON(b []byte) error {
unmarshalled := make(map[string]uint64)
err := json.Unmarshal(b, &unmarshalled)
if err != nil {
return err
}
*n = make(map[address.Address]uint64)
for saddr, nonce := range unmarshalled {
a, err := address.NewFromString(saddr)
if err != nil {
return err
}
(*n)[a] = nonce
}
return nil
}
func (m *MsgUuidMapType) MarshalJSON() ([]byte, error) {
marshalled := make(map[string]*types.SignedMessage)
for u, msg := range *m {
marshalled[u.String()] = msg
}
return json.Marshal(marshalled)
}
func (m *MsgUuidMapType) UnmarshalJSON(b []byte) error {
unmarshalled := make(map[string]*types.SignedMessage)
err := json.Unmarshal(b, &unmarshalled)
if err != nil {
return err
}
*m = make(map[uuid.UUID]*types.SignedMessage)
for suid, msg := range unmarshalled {
u, err := uuid.Parse(suid)
if err != nil {
return err
}
(*m)[u] = msg
}
return nil
}
// ChainExportConfig holds configuration for chain ranged exports.
type ChainExportConfig struct {
WriteBufferSize int
NumWorkers int
IncludeMessages bool
IncludeReceipts bool
IncludeStateRoots bool
}
-5
View File
@@ -1,5 +0,0 @@
package apitypes
type Aliaser interface {
AliasMethod(alias, original string)
}
+20 -45
View File
@@ -3,27 +3,22 @@ package v0api
import (
"context"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/paych"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/ipfs/go-cid"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -114,9 +109,6 @@ type FullNode interface {
// ChainDeleteObj deletes node referenced by the given CID
ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
// ChainPutObj puts and object into the blockstore
ChainPutObj(context.Context, blocks.Block) error
// ChainHasObj checks if a given CID exists in the chain blockstore.
ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
@@ -141,7 +133,7 @@ type FullNode interface {
// ChainGetPath returns a set of revert/apply operations needed to get from
// one tipset to another, for example:
// ```
//```
// to
// ^
// from tAA
@@ -150,7 +142,7 @@ type FullNode interface {
// ^---*--^
// ^
// tRR
// ```
//```
// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) //perm:read
@@ -367,12 +359,12 @@ type FullNode interface {
ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
// ClientUnimport removes references to the specified file from filestore
// ClientUnimport(path string)
//ClientUnimport(path string)
// ClientListImports lists imported files and their root CIDs
ClientListImports(ctx context.Context) ([]api.Import, error) //perm:write
// ClientListAsks() []Ask
//ClientListAsks() []Ask
// MethodGroup: State
// The State methods are used to query, inspect, and interact with chain state.
@@ -387,7 +379,7 @@ type FullNode interface {
StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
//
// If a tipset key is provided, and a replacing message is not found on chain,
// If a tipset key is provided, and a replacing message is found on chain,
// the method will return an error saying that the message wasn't found
//
// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -424,7 +416,7 @@ type FullNode interface {
// StateMinerPower returns the power of the indicated miner
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) //perm:read
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
// StateMinerPartitions returns all partitions in the specified deadline
@@ -450,9 +442,9 @@ type FullNode interface {
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) //perm:read
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) //perm:read
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
//
// NOTE: If a replacing message is found on chain, this method will return
@@ -529,19 +521,9 @@ type FullNode interface {
// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
// StateMarketDeals returns information about every deal in the Storage Market
StateMarketDeals(context.Context, types.TipSetKey) (map[string]*api.MarketDeal, error) //perm:read
StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
// StateMarketStorageDeal returns information about the indicated deal
StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal.
StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
// StateGetAllocation returns the allocation for a given address and allocation ID.
StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
// StateGetAllocations returns the all the allocations for a given client.
StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
// StateGetClaim returns the claim for a given address and claim ID.
StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
// StateGetClaims returns the all the claims for a given provider.
StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
// StateLookupID retrieves the ID address of the given address
StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
// StateAccountKey returns the public key address of the given ID address
@@ -602,7 +584,7 @@ type FullNode interface {
// Returns nil if there is no entry in the data cap table for the
// address.
StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
// StateVerifiedClientStatus returns the address of the Verified Registry's root key
StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
// can issue. It takes the deal size and verified status as parameters.
@@ -616,19 +598,12 @@ type FullNode interface {
StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
// StateNetworkVersion returns the network version at the given tipset
StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
// StateGetRandomnessFromTickets is used to sample the chain for randomness.
StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
// StateGetNetworkParams return current network params
StateGetNetworkParams(ctx context.Context) (*api.NetworkParams, error) //perm:read
// MethodGroup: Msig
// The Msig methods are used to interact with multisig wallets on the
// filecoin network
@@ -641,14 +616,14 @@ type FullNode interface {
// It takes the following params: <multisig address>, <start epoch>, <end epoch>
MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
// MsigGetPending returns pending transactions for the given multisig
// wallet. Once pending transactions are fully approved, they will no longer
// appear here.
//MsigGetPending returns pending transactions for the given multisig
//wallet. Once pending transactions are fully approved, they will no longer
//appear here.
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) //perm:read
// MsigCreate creates a multisig wallet
// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
// <initial balance>, <sender address of the create msg>, <gas price>
//<initial balance>, <sender address of the create msg>, <gas price>
MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
// MsigPropose proposes a multisig message
// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
+2 -9
View File
@@ -4,17 +4,15 @@ import (
"context"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
"github.com/filecoin-project/go-state-types/dline"
abinetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
// MODIFYING THE API INTERFACE
@@ -36,7 +34,6 @@ import (
type Gateway interface {
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
ChainHead(ctx context.Context) (*types.TipSet, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
@@ -45,25 +42,21 @@ type Gateway interface {
ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
StateNetworkName(context.Context) (dtypes.NetworkName, error)
StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-1
View File
@@ -2,7 +2,6 @@ package v0api
import (
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/api"
)
+236 -427
View File
File diff suppressed because it is too large Load Diff
+25 -178
View File
@@ -9,39 +9,35 @@ import (
reflect "reflect"
time "time"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
cid "github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
metrics "github.com/libp2p/go-libp2p/core/metrics"
network0 "github.com/libp2p/go-libp2p/core/network"
peer "github.com/libp2p/go-libp2p/core/peer"
protocol "github.com/libp2p/go-libp2p/core/protocol"
address "github.com/filecoin-project/go-address"
bitfield "github.com/filecoin-project/go-bitfield"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
datatransfer "github.com/filecoin-project/go-data-transfer"
retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
auth "github.com/filecoin-project/go-jsonrpc/auth"
abi "github.com/filecoin-project/go-state-types/abi"
big "github.com/filecoin-project/go-state-types/big"
paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
miner "github.com/filecoin-project/go-state-types/builtin/v9/miner"
verifreg "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
crypto "github.com/filecoin-project/go-state-types/crypto"
dline "github.com/filecoin-project/go-state-types/dline"
network "github.com/filecoin-project/go-state-types/network"
api "github.com/filecoin-project/lotus/api"
apitypes "github.com/filecoin-project/lotus/api/types"
v0api "github.com/filecoin-project/lotus/api/v0api"
miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
types "github.com/filecoin-project/lotus/chain/types"
alerting "github.com/filecoin-project/lotus/journal/alerting"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
imports "github.com/filecoin-project/lotus/node/repo/imports"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock"
uuid "github.com/google/uuid"
cid "github.com/ipfs/go-cid"
metrics "github.com/libp2p/go-libp2p-core/metrics"
network0 "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"
protocol "github.com/libp2p/go-libp2p-core/protocol"
)
// MockFullNode is a mock of FullNode interface.
@@ -381,20 +377,6 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
}
// ChainPutObj mocks base method.
func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// ChainPutObj indicates an expected call of ChainPutObj.
func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
}
// ChainReadObj mocks base method.
func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
m.ctrl.T.Helper()
@@ -2158,21 +2140,6 @@ func (mr *MockFullNodeMockRecorder) Shutdown(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockFullNode)(nil).Shutdown), arg0)
}
// StartTime mocks base method.
func (m *MockFullNode) StartTime(arg0 context.Context) (time.Time, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StartTime", arg0)
ret0, _ := ret[0].(time.Time)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StartTime indicates an expected call of StartTime.
func (mr *MockFullNodeMockRecorder) StartTime(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartTime", reflect.TypeOf((*MockFullNode)(nil).StartTime), arg0)
}
// StateAccountKey mocks base method.
func (m *MockFullNode) StateAccountKey(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (address.Address, error) {
m.ctrl.T.Helper()
@@ -2188,36 +2155,6 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
}
// StateActorCodeCIDs mocks base method.
func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
ret0, _ := ret[0].(map[string]cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
}
// StateActorManifestCID mocks base method.
func (m *MockFullNode) StateActorManifestCID(arg0 context.Context, arg1 network.Version) (cid.Cid, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateActorManifestCID", arg0, arg1)
ret0, _ := ret[0].(cid.Cid)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateActorManifestCID indicates an expected call of StateActorManifestCID.
func (mr *MockFullNodeMockRecorder) StateActorManifestCID(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorManifestCID", reflect.TypeOf((*MockFullNode)(nil).StateActorManifestCID), arg0, arg1)
}
// StateAllMinerFaults mocks base method.
func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
m.ctrl.T.Helper()
@@ -2249,10 +2186,10 @@ func (mr *MockFullNodeMockRecorder) StateCall(arg0, arg1, arg2 interface{}) *gom
}
// StateChangedActors mocks base method.
func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.ActorV5, error) {
func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.Actor, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateChangedActors", arg0, arg1, arg2)
ret0, _ := ret[0].(map[string]types.ActorV5)
ret0, _ := ret[0].(map[string]types.Actor)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2324,10 +2261,10 @@ func (mr *MockFullNodeMockRecorder) StateDecodeParams(arg0, arg1, arg2, arg3, ar
}
// StateGetActor mocks base method.
func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.ActorV5, error) {
func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.Actor, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetActor", arg0, arg1, arg2)
ret0, _ := ret[0].(*types.ActorV5)
ret0, _ := ret[0].(*types.Actor)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2338,96 +2275,6 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
}
// StateGetAllocation mocks base method.
func (m *MockFullNode) StateGetAllocation(arg0 context.Context, arg1 address.Address, arg2 verifreg.AllocationId, arg3 types.TipSetKey) (*verifreg.Allocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetAllocation", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*verifreg.Allocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetAllocation indicates an expected call of StateGetAllocation.
func (mr *MockFullNodeMockRecorder) StateGetAllocation(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocation", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocation), arg0, arg1, arg2, arg3)
}
// StateGetAllocationForPendingDeal mocks base method.
func (m *MockFullNode) StateGetAllocationForPendingDeal(arg0 context.Context, arg1 abi.DealID, arg2 types.TipSetKey) (*verifreg.Allocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetAllocationForPendingDeal", arg0, arg1, arg2)
ret0, _ := ret[0].(*verifreg.Allocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetAllocationForPendingDeal indicates an expected call of StateGetAllocationForPendingDeal.
func (mr *MockFullNodeMockRecorder) StateGetAllocationForPendingDeal(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocationForPendingDeal", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocationForPendingDeal), arg0, arg1, arg2)
}
// StateGetAllocations mocks base method.
func (m *MockFullNode) StateGetAllocations(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.AllocationId]verifreg.Allocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetAllocations", arg0, arg1, arg2)
ret0, _ := ret[0].(map[verifreg.AllocationId]verifreg.Allocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetAllocations indicates an expected call of StateGetAllocations.
func (mr *MockFullNodeMockRecorder) StateGetAllocations(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocations", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocations), arg0, arg1, arg2)
}
// StateGetClaim mocks base method.
func (m *MockFullNode) StateGetClaim(arg0 context.Context, arg1 address.Address, arg2 verifreg.ClaimId, arg3 types.TipSetKey) (*verifreg.Claim, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetClaim", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*verifreg.Claim)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetClaim indicates an expected call of StateGetClaim.
func (mr *MockFullNodeMockRecorder) StateGetClaim(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaim", reflect.TypeOf((*MockFullNode)(nil).StateGetClaim), arg0, arg1, arg2, arg3)
}
// StateGetClaims mocks base method.
func (m *MockFullNode) StateGetClaims(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.ClaimId]verifreg.Claim, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetClaims", arg0, arg1, arg2)
ret0, _ := ret[0].(map[verifreg.ClaimId]verifreg.Claim)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetClaims indicates an expected call of StateGetClaims.
func (mr *MockFullNodeMockRecorder) StateGetClaims(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaims", reflect.TypeOf((*MockFullNode)(nil).StateGetClaims), arg0, arg1, arg2)
}
// StateGetNetworkParams mocks base method.
func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateGetNetworkParams", arg0)
ret0, _ := ret[0].(*api.NetworkParams)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StateGetNetworkParams indicates an expected call of StateGetNetworkParams.
func (mr *MockFullNodeMockRecorder) StateGetNetworkParams(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetNetworkParams", reflect.TypeOf((*MockFullNode)(nil).StateGetNetworkParams), arg0)
}
// StateGetRandomnessFromBeacon mocks base method.
func (m *MockFullNode) StateGetRandomnessFromBeacon(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
m.ctrl.T.Helper()
@@ -2549,10 +2396,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
}
// StateMarketDeals mocks base method.
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
ret0, _ := ret[0].(map[string]*api.MarketDeal)
ret0, _ := ret[0].(map[string]api.MarketDeal)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2654,10 +2501,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
}
// StateMinerInfo mocks base method.
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
ret0, _ := ret[0].(api.MinerInfo)
ret0, _ := ret[0].(miner.MinerInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2669,7 +2516,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
}
// StateMinerInitialPledgeCollateral mocks base method.
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2714,7 +2561,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
}
// StateMinerPreCommitDepositForPower mocks base method.
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(big.Int)
@@ -2894,10 +2741,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsgLimited(arg0, arg1, arg2 inter
}
// StateSectorExpiration mocks base method.
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorExpiration)
ret0, _ := ret[0].(*miner.SectorExpiration)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@@ -2924,10 +2771,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
}
// StateSectorPartition mocks base method.
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(*miner0.SectorLocation)
ret0, _ := ret[0].(*miner.SectorLocation)
ret1, _ := ret[1].(error)
return ret0, ret1
}
+12 -26
View File
@@ -3,40 +3,30 @@ package v0api
import (
"context"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p/core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/types"
"golang.org/x/xerrors"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin/v9/miner"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/v1api"
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
)
type WrapperV1Full struct {
v1api.FullNode
}
func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
if err != nil {
return miner.SectorPreCommitOnChainInfo{}, err
}
if pi == nil {
return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
}
return *pi, nil
}
func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
}
@@ -362,8 +352,4 @@ func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner add
return a.Response, nil
}
func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
return w.StateGetBeaconEntry(ctx, epoch)
}
var _ FullNode = &WrapperV1Full{}
-2
View File
@@ -7,8 +7,6 @@ import (
type FullNode = api.FullNode
type FullNodeStruct = api.FullNodeStruct
type RawFullNodeAPI FullNode
func PermissionedFullAPI(a FullNode) FullNode {
return api.PermissionedFullAPI(a)
}
+3 -3
View File
@@ -3,7 +3,7 @@ package api
import (
"fmt"
"golang.org/x/xerrors"
xerrors "golang.org/x/xerrors"
)
type Version uint32
@@ -55,10 +55,10 @@ func VersionForType(nodeType NodeType) (Version, error) {
// semver versions of the rpc api exposed
var (
FullAPIVersion0 = newVer(1, 5, 0)
FullAPIVersion1 = newVer(2, 3, 0)
FullAPIVersion1 = newVer(2, 2, 0)
MinerAPIVersion0 = newVer(1, 5, 0)
WorkerAPIVersion0 = newVer(1, 7, 0)
WorkerAPIVersion0 = newVer(1, 6, 0)
)
//nolint:varcheck,deadcode
+5 -12
View File
@@ -3,15 +3,14 @@ package blockstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"golang.org/x/xerrors"
)
type ChainIO interface {
ChainReadObj(context.Context, cid.Cid) ([]byte, error)
ChainHasObj(context.Context, cid.Cid) (bool, error)
ChainPutObj(context.Context, blocks.Block) error
}
type apiBlockstore struct {
@@ -50,18 +49,12 @@ func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
return len(bb), nil
}
func (a *apiBlockstore) Put(ctx context.Context, block blocks.Block) error {
return a.api.ChainPutObj(ctx, block)
func (a *apiBlockstore) Put(context.Context, blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
for _, block := range blocks {
err := a.api.ChainPutObj(ctx, block)
if err != nil {
return err
}
}
return nil
func (a *apiBlockstore) PutMany(context.Context, []blocks.Block) error {
return xerrors.New("not supported")
}
func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+7 -11
View File
@@ -5,10 +5,10 @@ import (
"sync"
"time"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
block "github.com/ipfs/go-libipfs/blocks"
"golang.org/x/xerrors"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
// autolog is a logger for the autobatching blockstore. It is subscoped from the
@@ -176,27 +176,23 @@ func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
return blk, nil
}
if !ipld.IsNotFound(err) {
if err != ErrNotFound {
return blk, err
}
bs.stateLock.Lock()
defer bs.stateLock.Unlock()
v, ok := bs.flushingBatch.blockMap[c]
if ok {
bs.stateLock.Unlock()
return v, nil
}
v, ok = bs.bufferedBatch.blockMap[c]
if ok {
bs.stateLock.Unlock()
return v, nil
}
bs.stateLock.Unlock()
// We have to check the backing store one more time because it may have been flushed by the
// time we were able to take the lock above.
return bs.backingBs.Get(ctx, c)
return bs.Get(ctx, c)
}
func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
@@ -218,7 +214,7 @@ func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
if err == nil {
return true, nil
}
if ipld.IsNotFound(err) {
if err == ErrNotFound {
return false, nil
}
-5
View File
@@ -4,7 +4,6 @@ import (
"context"
"testing"
ipld "github.com/ipfs/go-ipld-format"
"github.com/stretchr/testify/require"
)
@@ -30,10 +29,6 @@ func TestAutobatchBlockstore(t *testing.T) {
require.NoError(t, err)
require.Equal(t, b2.RawData(), v2.RawData())
// Regression test for a deadlock.
_, err = ab.Get(ctx, b3.Cid())
require.True(t, ipld.IsNotFound(err))
require.NoError(t, ab.Flush(ctx))
require.NoError(t, ab.Shutdown(ctx))
}
+15 -94
View File
@@ -13,14 +13,13 @@ import (
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"github.com/dgraph-io/badger/v2/pb"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
logger "github.com/ipfs/go-log/v2"
pool "github.com/libp2p/go-buffer-pool"
"github.com/multiformats/go-base32"
"go.uber.org/zap"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
logger "github.com/ipfs/go-log/v2"
pool "github.com/libp2p/go-buffer-pool"
"github.com/filecoin-project/lotus/blockstore"
)
@@ -45,8 +44,7 @@ const (
// MemoryMap is equivalent to badger/options.MemoryMap.
MemoryMap = options.MemoryMap
// LoadToRAM is equivalent to badger/options.LoadToRAM.
LoadToRAM = options.LoadToRAM
defaultGCThreshold = 0.125
LoadToRAM = options.LoadToRAM
)
// Options embeds the badger options themselves, and augments them with
@@ -396,9 +394,6 @@ func (b *Blockstore) doCopy(from, to *badger.DB) error {
if workers < 2 {
workers = 2
}
if workers > 8 {
workers = 8
}
stream := from.NewStream()
stream.NumGo = workers
@@ -444,7 +439,7 @@ func (b *Blockstore) deleteDB(path string) {
}
}
func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq time.Duration, check func() error) error {
func (b *Blockstore) onlineGC() error {
b.lockDB()
defer b.unlockDB()
@@ -453,26 +448,14 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
if nworkers < 2 {
nworkers = 2
}
if nworkers > 7 { // max out at 1 goroutine per badger level
nworkers = 7
}
err := b.db.Flatten(nworkers)
if err != nil {
return err
}
checkTick := time.NewTimer(checkFreq)
defer checkTick.Stop()
for err == nil {
select {
case <-ctx.Done():
err = ctx.Err()
case <-checkTick.C:
err = check()
checkTick.Reset(checkFreq)
default:
err = b.db.RunValueLogGC(threshold)
}
err = b.db.RunValueLogGC(0.125)
}
if err == badger.ErrNoRewrite {
@@ -485,7 +468,7 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
// CollectGarbage compacts and runs garbage collection on the value log;
// implements the BlockstoreGC trait
func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
if err := b.access(); err != nil {
return err
}
@@ -502,58 +485,8 @@ func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.Bloc
if options.FullGC {
return b.movingGC()
}
threshold := options.Threshold
if threshold == 0 {
threshold = defaultGCThreshold
}
checkFreq := options.CheckFreq
if checkFreq < 30*time.Second { // disallow checking more frequently than block time
checkFreq = 30 * time.Second
}
check := options.Check
if check == nil {
check = func() error {
return nil
}
}
return b.onlineGC(ctx, threshold, checkFreq, check)
}
// GCOnce runs garbage collection on the value log;
// implements BlockstoreGCOnce trait
func (b *Blockstore) GCOnce(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
var options blockstore.BlockstoreGCOptions
for _, opt := range opts {
err := opt(&options)
if err != nil {
return err
}
}
if options.FullGC {
return xerrors.Errorf("FullGC option specified for GCOnce but full GC is non incremental")
}
threshold := options.Threshold
if threshold == 0 {
threshold = defaultGCThreshold
}
b.lockDB()
defer b.unlockDB()
// Note no compaction needed before single GC as we will hit at most one vlog anyway
err := b.db.RunValueLogGC(threshold)
if err == badger.ErrNoRewrite {
// not really an error in this case, it signals the end of GC
return nil
}
return err
return b.onlineGC()
}
// Size returns the aggregate size of the blockstore
@@ -611,25 +544,13 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
case nil:
return item.Value(fn)
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to view block from badger blockstore: %w", err)
}
})
}
func (b *Blockstore) Flush(context.Context) error {
if err := b.access(); err != nil {
return err
}
defer b.viewers.Done()
b.lockDB()
defer b.unlockDB()
return b.db.Sync()
}
// Has implements Blockstore.Has.
func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
if err := b.access(); err != nil {
@@ -663,7 +584,7 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
// Get implements Blockstore.Get.
func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
if !cid.Defined() {
return nil, ipld.ErrNotFound{Cid: cid}
return nil, blockstore.ErrNotFound
}
if err := b.access(); err != nil {
@@ -686,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
val, err = item.ValueCopy(nil)
return err
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to get block from badger blockstore: %w", err)
}
@@ -718,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
case nil:
size = int(item.ValueSize())
case badger.ErrKeyNotFound:
return ipld.ErrNotFound{Cid: cid}
return blockstore.ErrNotFound
default:
return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
}
+6 -5
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package badgerbs
import (
@@ -10,11 +10,12 @@ import (
"strings"
"testing"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/blockstore"
)
@@ -145,7 +146,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
return nil
})
g.Go(func() error {
return db.CollectGarbage(ctx, blockstore.WithFullGC(true))
return db.CollectGarbage(blockstore.WithFullGC(true))
})
err = g.Wait()
@@ -230,7 +231,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
checkPath()
// now do another FullGC to test the double move and following of symlinks
if err := db.CollectGarbage(ctx, blockstore.WithFullGC(true)); err != nil {
if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
t.Fatal(err)
}
+8 -9
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package badgerbs
import (
@@ -9,13 +9,13 @@ import (
"strings"
"testing"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
u "github.com/ipfs/go-ipfs-util"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/blockstore"
"github.com/stretchr/testify/require"
)
// TODO: move this to go-ipfs-blockstore.
@@ -56,7 +56,7 @@ func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
c := cid.NewCidV0(u.Hash([]byte("stuff")))
bl, err := bs.Get(ctx, c)
require.Nil(t, bl)
require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
require.Equal(t, blockstore.ErrNotFound, err)
}
func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
@@ -69,7 +69,7 @@ func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
}
_, err := bs.Get(ctx, cid.Undef)
require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
require.Equal(t, blockstore.ErrNotFound, err)
}
func (s *Suite) TestPutThenGetBlock(t *testing.T) {
@@ -164,9 +164,8 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
require.NoError(t, err)
require.Zero(t, emptySize)
missingCid := missingBlock.Cid()
missingSize, err := bs.GetSize(ctx, missingCid)
require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
require.Equal(t, blockstore.ErrNotFound, err)
require.Equal(t, -1, missingSize)
}
+6 -48
View File
@@ -2,16 +2,18 @@ package blockstore
import (
"context"
"time"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
ds "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log/v2"
blockstore "github.com/ipfs/go-ipfs-blockstore"
)
var log = logging.Logger("blockstore")
var ErrNotFound = blockstore.ErrNotFound
// Blockstore is the blockstore interface used by Lotus. It is the union
// of the basic go-ipfs blockstore, with other capabilities required by Lotus,
// e.g. View or Sync.
@@ -19,7 +21,6 @@ type Blockstore interface {
blockstore.Blockstore
blockstore.Viewer
BatchDeleter
Flusher
}
// BasicBlockstore is an alias to the original IPFS Blockstore.
@@ -27,10 +28,6 @@ type BasicBlockstore = blockstore.Blockstore
type Viewer = blockstore.Viewer
type Flusher interface {
Flush(context.Context) error
}
type BatchDeleter interface {
DeleteMany(ctx context.Context, cids []cid.Cid) error
}
@@ -42,12 +39,7 @@ type BlockstoreIterator interface {
// BlockstoreGC is a trait for blockstores that support online garbage collection
type BlockstoreGC interface {
CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error
}
// BlockstoreGCOnce is a trait for a blockstore that supports incremental online garbage collection
type BlockstoreGCOnce interface {
GCOnce(ctx context.Context, options ...BlockstoreGCOption) error
CollectGarbage(options ...BlockstoreGCOption) error
}
// BlockstoreGCOption is a functional interface for controlling blockstore GC options
@@ -56,12 +48,6 @@ type BlockstoreGCOption = func(*BlockstoreGCOptions) error
// BlockstoreGCOptions is a struct with GC options
type BlockstoreGCOptions struct {
FullGC bool
// fraction of garbage in badger vlog before its worth processing in online GC
Threshold float64
// how often to call the check function
CheckFreq time.Duration
// function to call periodically to pause or early terminate GC
Check func() error
}
func WithFullGC(fullgc bool) BlockstoreGCOption {
@@ -71,27 +57,6 @@ func WithFullGC(fullgc bool) BlockstoreGCOption {
}
}
func WithThreshold(threshold float64) BlockstoreGCOption {
return func(opts *BlockstoreGCOptions) error {
opts.Threshold = threshold
return nil
}
}
func WithCheckFreq(f time.Duration) BlockstoreGCOption {
return func(opts *BlockstoreGCOptions) error {
opts.CheckFreq = f
return nil
}
}
func WithCheck(check func() error) BlockstoreGCOption {
return func(opts *BlockstoreGCOptions) error {
opts.Check = check
return nil
}
}
// BlockstoreSize is a trait for on-disk blockstores that can report their size
type BlockstoreSize interface {
Size() (int64, error)
@@ -130,13 +95,6 @@ type adaptedBlockstore struct {
var _ Blockstore = (*adaptedBlockstore)(nil)
func (a *adaptedBlockstore) Flush(ctx context.Context) error {
if flusher, canFlush := a.Blockstore.(Flusher); canFlush {
return flusher.Flush(ctx)
}
return nil
}
func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
blk, err := a.Get(ctx, cid)
if err != nil {
+4 -7
View File
@@ -4,9 +4,8 @@ import (
"context"
"os"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
block "github.com/ipfs/go-libipfs/blocks"
)
// buflog is a logger for the buffered blockstore. It is subscoped from the
@@ -46,8 +45,6 @@ var (
_ Viewer = (*BufferedBlockstore)(nil)
)
func (bs *BufferedBlockstore) Flush(ctx context.Context) error { return bs.write.Flush(ctx) }
func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
a, err := bs.read.AllKeysChan(ctx)
if err != nil {
@@ -109,7 +106,7 @@ func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) er
func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
// both stores are viewable.
if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
// not found in write blockstore; fall through.
} else {
return err // propagate errors, or nil, i.e. found.
@@ -119,7 +116,7 @@ func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func
func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
if out, err := bs.write.Get(ctx, c); err != nil {
if !ipld.IsNotFound(err) {
if err != ErrNotFound {
return nil, err
}
} else {
@@ -131,7 +128,7 @@ func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
s, err := bs.read.GetSize(ctx, c)
if ipld.IsNotFound(err) || s == 0 {
if err == ErrNotFound || s == 0 {
return bs.write.GetSize(ctx, c)
}
-441
View File
@@ -1,441 +0,0 @@
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
package blockstore
import (
"fmt"
"io"
"math"
"sort"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
var _ = xerrors.Errorf
var _ = cid.Undef
var _ = math.E
var _ = sort.Sort
var lengthBufNetRpcReq = []byte{132}
func (t *NetRpcReq) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write(lengthBufNetRpcReq); err != nil {
return err
}
// t.Type (blockstore.NetRPCReqType) (uint8)
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
return err
}
// t.ID (uint64) (uint64)
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
return err
}
// t.Cid ([]cid.Cid) (slice)
if len(t.Cid) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Cid was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Cid))); err != nil {
return err
}
for _, v := range t.Cid {
if err := cbg.WriteCid(w, v); err != nil {
return xerrors.Errorf("failed writing cid field t.Cid: %w", err)
}
}
// t.Data ([][]uint8) (slice)
if len(t.Data) > cbg.MaxLength {
return xerrors.Errorf("Slice value in field t.Data was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Data))); err != nil {
return err
}
for _, v := range t.Data {
if len(v) > cbg.ByteArrayMaxLen {
return xerrors.Errorf("Byte array in field v was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(v))); err != nil {
return err
}
if _, err := cw.Write(v[:]); err != nil {
return err
}
}
return nil
}
func (t *NetRpcReq) UnmarshalCBOR(r io.Reader) (err error) {
*t = NetRpcReq{}
cr := cbg.NewCborReader(r)
maj, extra, err := cr.ReadHeader()
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Type (blockstore.NetRPCReqType) (uint8)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint8 field")
}
if extra > math.MaxUint8 {
return fmt.Errorf("integer in input was too large for uint8 field")
}
t.Type = NetRPCReqType(extra)
// t.ID (uint64) (uint64)
{
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ID = uint64(extra)
}
// t.Cid ([]cid.Cid) (slice)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.Cid: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Cid = make([]cid.Cid, extra)
}
for i := 0; i < int(extra); i++ {
c, err := cbg.ReadCid(cr)
if err != nil {
return xerrors.Errorf("reading cid field t.Cid failed: %w", err)
}
t.Cid[i] = c
}
// t.Data ([][]uint8) (slice)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if extra > cbg.MaxLength {
return fmt.Errorf("t.Data: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Data = make([][]uint8, extra)
}
for i := 0; i < int(extra); i++ {
{
var maj byte
var extra uint64
var err error
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if extra > cbg.ByteArrayMaxLen {
return fmt.Errorf("t.Data[i]: byte array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
if extra > 0 {
t.Data[i] = make([]uint8, extra)
}
if _, err := io.ReadFull(cr, t.Data[i][:]); err != nil {
return err
}
}
}
return nil
}
var lengthBufNetRpcResp = []byte{131}
func (t *NetRpcResp) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write(lengthBufNetRpcResp); err != nil {
return err
}
// t.Type (blockstore.NetRPCRespType) (uint8)
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
return err
}
// t.ID (uint64) (uint64)
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
return err
}
// t.Data ([]uint8) (slice)
if len(t.Data) > cbg.ByteArrayMaxLen {
return xerrors.Errorf("Byte array in field t.Data was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Data))); err != nil {
return err
}
if _, err := cw.Write(t.Data[:]); err != nil {
return err
}
return nil
}
func (t *NetRpcResp) UnmarshalCBOR(r io.Reader) (err error) {
*t = NetRpcResp{}
cr := cbg.NewCborReader(r)
maj, extra, err := cr.ReadHeader()
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Type (blockstore.NetRPCRespType) (uint8)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint8 field")
}
if extra > math.MaxUint8 {
return fmt.Errorf("integer in input was too large for uint8 field")
}
t.Type = NetRPCRespType(extra)
// t.ID (uint64) (uint64)
{
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ID = uint64(extra)
}
// t.Data ([]uint8) (slice)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if extra > cbg.ByteArrayMaxLen {
return fmt.Errorf("t.Data: byte array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
if extra > 0 {
t.Data = make([]uint8, extra)
}
if _, err := io.ReadFull(cr, t.Data[:]); err != nil {
return err
}
return nil
}
var lengthBufNetRpcErr = []byte{131}
func (t *NetRpcErr) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
cw := cbg.NewCborWriter(w)
if _, err := cw.Write(lengthBufNetRpcErr); err != nil {
return err
}
// t.Type (blockstore.NetRPCErrType) (uint8)
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
return err
}
// t.Msg (string) (string)
if len(t.Msg) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Msg was too long")
}
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Msg))); err != nil {
return err
}
if _, err := io.WriteString(w, string(t.Msg)); err != nil {
return err
}
// t.Cid (cid.Cid) (struct)
if t.Cid == nil {
if _, err := cw.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(cw, *t.Cid); err != nil {
return xerrors.Errorf("failed to write cid field t.Cid: %w", err)
}
}
return nil
}
func (t *NetRpcErr) UnmarshalCBOR(r io.Reader) (err error) {
*t = NetRpcErr{}
cr := cbg.NewCborReader(r)
maj, extra, err := cr.ReadHeader()
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Type (blockstore.NetRPCErrType) (uint8)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint8 field")
}
if extra > math.MaxUint8 {
return fmt.Errorf("integer in input was too large for uint8 field")
}
t.Type = NetRPCErrType(extra)
// t.Msg (string) (string)
{
sval, err := cbg.ReadString(cr)
if err != nil {
return err
}
t.Msg = string(sval)
}
// t.Cid (cid.Cid) (struct)
{
b, err := cr.ReadByte()
if err != nil {
return err
}
if b != cbg.CborNull[0] {
if err := cr.UnreadByte(); err != nil {
return err
}
c, err := cbg.ReadCid(cr)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Cid: %w", err)
}
t.Cid = &c
}
}
return nil
}
+2 -6
View File
@@ -4,8 +4,8 @@ import (
"context"
"io"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
var _ Blockstore = (*discardstore)(nil)
@@ -38,10 +38,6 @@ func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) err
return b.bs.View(ctx, cid, f)
}
func (b *discardstore) Flush(ctx context.Context) error {
return nil
}
func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
return nil
}
+10 -10
View File
@@ -5,10 +5,10 @@ import (
"sync"
"time"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
// UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
if fbs.missFn == nil {
log.Errorw("fallbackstore: missFn not configured yet")
return nil, ipld.ErrNotFound{Cid: c}
return nil, ErrNotFound
}
}
@@ -79,10 +79,10 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
b, err := fbs.Blockstore.Get(ctx, c)
switch {
case err == nil:
switch err {
case nil:
return b, nil
case ipld.IsNotFound(err):
case ErrNotFound:
return fbs.getFallback(c)
default:
return b, err
@@ -91,10 +91,10 @@ func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, err
func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
sz, err := fbs.Blockstore.GetSize(ctx, c)
switch {
case err == nil:
switch err {
case nil:
return sz, nil
case ipld.IsNotFound(err):
case ErrNotFound:
b, err := fbs.getFallback(c)
if err != nil {
return 0, err
+4 -15
View File
@@ -4,10 +4,11 @@ import (
"context"
"io"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)
var _ Blockstore = (*idstore)(nil)
@@ -102,14 +103,6 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
return b.bs.Put(ctx, blk)
}
func (b *idstore) ForEachKey(f func(cid.Cid) error) error {
iterBstore, ok := b.bs.(BlockstoreIterator)
if !ok {
return xerrors.Errorf("underlying blockstore (type %T) doesn't support fast iteration", b.bs)
}
return iterBstore.ForEachKey(f)
}
func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
toPut := make([]blocks.Block, 0, len(blks))
for _, blk := range blks {
@@ -179,7 +172,3 @@ func (b *idstore) Close() error {
}
return nil
}
func (b *idstore) Flush(ctx context.Context) error {
return b.bs.Flush(ctx)
}
+9 -7
View File
@@ -3,17 +3,19 @@ package blockstore
import (
"bytes"
"context"
"io"
"io/ioutil"
"golang.org/x/xerrors"
"github.com/multiformats/go-multiaddr"
"github.com/multiformats/go-multihash"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
httpapi "github.com/ipfs/go-ipfs-http-client"
blocks "github.com/ipfs/go-libipfs/blocks"
iface "github.com/ipfs/interface-go-ipfs-core"
"github.com/ipfs/interface-go-ipfs-core/options"
"github.com/ipfs/interface-go-ipfs-core/path"
"github.com/multiformats/go-multiaddr"
"github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
)
type IPFSBlockstore struct {
@@ -103,7 +105,7 @@ func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, er
return nil, xerrors.Errorf("getting ipfs block: %w", err)
}
data, err := io.ReadAll(rd)
data, err := ioutil.ReadAll(rd)
if err != nil {
return nil, err
}
@@ -128,7 +130,7 @@ func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
options.Block.Hash(mhd.Code, mhd.Length),
options.Block.Format(multihash.Codes[block.Cid().Type()]))
options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
return err
}
+15 -22
View File
@@ -3,9 +3,8 @@ package blockstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
)
// NewMemory returns a temporary memory-backed blockstore.
@@ -14,52 +13,46 @@ func NewMemory() MemBlockstore {
}
// MemBlockstore is a terminal blockstore that keeps blocks in memory.
// To match behavior of badger blockstore we index by multihash only.
type MemBlockstore map[string]blocks.Block
func (MemBlockstore) Flush(context.Context) error { return nil }
type MemBlockstore map[cid.Cid]blocks.Block
func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
delete(m, string(k.Hash()))
delete(m, k)
return nil
}
func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
for _, k := range ks {
delete(m, string(k.Hash()))
delete(m, k)
}
return nil
}
func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
_, ok := m[string(k.Hash())]
_, ok := m[k]
return ok, nil
}
func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
b, ok := m[string(k.Hash())]
b, ok := m[k]
if !ok {
return ipld.ErrNotFound{Cid: k}
return ErrNotFound
}
return callback(b.RawData())
}
func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
b, ok := m[string(k.Hash())]
b, ok := m[k]
if !ok {
return nil, ipld.ErrNotFound{Cid: k}
}
if b.Cid().Prefix().Codec != k.Prefix().Codec {
return blocks.NewBlockWithCid(b.RawData(), k)
return nil, ErrNotFound
}
return b, nil
}
// GetSize returns the CIDs mapped BlockSize
func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
b, ok := m[string(k.Hash())]
b, ok := m[k]
if !ok {
return 0, ipld.ErrNotFound{Cid: k}
return 0, ErrNotFound
}
return len(b.RawData()), nil
}
@@ -68,7 +61,7 @@ func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
// Convert to a basic block for safety, but try to reuse the existing
// block if it's already a basic block.
k := string(b.Cid().Hash())
k := b.Cid()
if _, ok := b.(*blocks.BasicBlock); !ok {
// If we already have the block, abort.
if _, ok := m[k]; ok {
@@ -77,7 +70,7 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
// the error is only for debugging.
b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
}
m[k] = b
m[b.Cid()] = b
return nil
}
@@ -95,8 +88,8 @@ func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
// the given context, closing the channel if it becomes Done.
func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
ch := make(chan cid.Cid, len(m))
for _, b := range m {
ch <- b.Cid()
for k := range m {
ch <- k
}
close(ch)
return ch, nil
-45
View File
@@ -1,45 +0,0 @@
package blockstore
import (
"context"
"testing"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
mh "github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
)
func TestMemGetCodec(t *testing.T) {
ctx := context.Background()
bs := NewMemory()
cborArr := []byte{0x82, 1, 2}
h, err := mh.Sum(cborArr, mh.SHA2_256, -1)
require.NoError(t, err)
rawCid := cid.NewCidV1(cid.Raw, h)
rawBlk, err := blocks.NewBlockWithCid(cborArr, rawCid)
require.NoError(t, err)
err = bs.Put(ctx, rawBlk)
require.NoError(t, err)
cborCid := cid.NewCidV1(cid.DagCBOR, h)
cborBlk, err := bs.Get(ctx, cborCid)
require.NoError(t, err)
require.Equal(t, cborCid.Prefix(), cborBlk.Cid().Prefix())
require.EqualValues(t, cborArr, cborBlk.RawData())
// was allocated
require.NotEqual(t, cborBlk, rawBlk)
gotRawBlk, err := bs.Get(ctx, rawCid)
require.NoError(t, err)
// not allocated
require.Equal(t, rawBlk, gotRawBlk)
}
-426
View File
@@ -1,426 +0,0 @@
package blockstore
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"sync"
"sync/atomic"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/libp2p/go-msgio"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
type NetRPCReqType byte
const (
NRpcHas NetRPCReqType = iota
NRpcGet
NRpcGetSize
NRpcPut
NRpcDelete
// todo cancel req
)
type NetRPCRespType byte
const (
NRpcOK NetRPCRespType = iota
NRpcErr
NRpcMore
)
type NetRPCErrType byte
const (
NRpcErrGeneric NetRPCErrType = iota
NRpcErrNotFound
)
type NetRpcReq struct {
Type NetRPCReqType
ID uint64
Cid []cid.Cid // todo maxsize?
Data [][]byte // todo maxsize?
}
type NetRpcResp struct {
Type NetRPCRespType
ID uint64
// error or cids in allkeys
Data []byte // todo maxsize?
next <-chan NetRpcResp
}
type NetRpcErr struct {
Type NetRPCErrType
Msg string
// in case of NRpcErrNotFound
Cid *cid.Cid
}
type NetworkStore struct {
// note: writer is thread-safe
msgStream msgio.ReadWriteCloser
// atomic
reqCount uint64
respLk sync.Mutex
// respMap is nil after store closes
respMap map[uint64]chan<- NetRpcResp
closing chan struct{}
closed chan struct{}
closeLk sync.Mutex
onClose []func()
}
func NewNetworkStore(mss msgio.ReadWriteCloser) *NetworkStore {
ns := &NetworkStore{
msgStream: mss,
respMap: map[uint64]chan<- NetRpcResp{},
closing: make(chan struct{}),
closed: make(chan struct{}),
}
go ns.receive()
return ns
}
func (n *NetworkStore) shutdown(msg string) {
if err := n.msgStream.Close(); err != nil {
log.Errorw("closing netstore msg stream", "error", err)
}
nerr := NetRpcErr{
Type: NRpcErrGeneric,
Msg: msg,
Cid: nil,
}
var errb bytes.Buffer
if err := nerr.MarshalCBOR(&errb); err != nil {
log.Errorw("netstore shutdown: error marshaling error", "err", err)
}
n.respLk.Lock()
for id, resps := range n.respMap {
resps <- NetRpcResp{
Type: NRpcErr,
ID: id,
Data: errb.Bytes(),
}
}
n.respMap = nil
n.respLk.Unlock()
}
func (n *NetworkStore) OnClose(cb func()) {
n.closeLk.Lock()
defer n.closeLk.Unlock()
select {
case <-n.closed:
cb()
default:
n.onClose = append(n.onClose, cb)
}
}
func (n *NetworkStore) receive() {
defer func() {
n.closeLk.Lock()
defer n.closeLk.Unlock()
close(n.closed)
if n.onClose != nil {
for _, f := range n.onClose {
f()
}
}
}()
for {
select {
case <-n.closing:
n.shutdown("netstore stopping")
return
default:
}
msg, err := n.msgStream.ReadMsg()
if err != nil {
n.shutdown(fmt.Sprintf("netstore ReadMsg: %s", err))
return
}
var resp NetRpcResp
if err := resp.UnmarshalCBOR(bytes.NewReader(msg)); err != nil {
n.shutdown(fmt.Sprintf("unmarshaling netstore response: %s", err))
return
}
n.msgStream.ReleaseMsg(msg)
n.respLk.Lock()
if ch, ok := n.respMap[resp.ID]; ok {
if resp.Type == NRpcMore {
nch := make(chan NetRpcResp, 1)
resp.next = nch
n.respMap[resp.ID] = nch
} else {
delete(n.respMap, resp.ID)
}
ch <- resp
}
n.respLk.Unlock()
}
}
func (n *NetworkStore) sendRpc(rt NetRPCReqType, cids []cid.Cid, data [][]byte) (uint64, <-chan NetRpcResp, error) {
rid := atomic.AddUint64(&n.reqCount, 1)
respCh := make(chan NetRpcResp, 1) // todo pool?
n.respLk.Lock()
if n.respMap == nil {
n.respLk.Unlock()
return 0, nil, xerrors.Errorf("netstore closed")
}
n.respMap[rid] = respCh
n.respLk.Unlock()
req := NetRpcReq{
Type: rt,
ID: rid,
Cid: cids,
Data: data,
}
var rbuf bytes.Buffer // todo buffer pool
if err := req.MarshalCBOR(&rbuf); err != nil {
n.respLk.Lock()
defer n.respLk.Unlock()
if n.respMap == nil {
return 0, nil, xerrors.Errorf("netstore closed")
}
delete(n.respMap, rid)
return 0, nil, err
}
if err := n.msgStream.WriteMsg(rbuf.Bytes()); err != nil {
n.respLk.Lock()
defer n.respLk.Unlock()
if n.respMap == nil {
return 0, nil, xerrors.Errorf("netstore closed")
}
delete(n.respMap, rid)
return 0, nil, err
}
return rid, respCh, nil
}
func (n *NetworkStore) waitResp(ctx context.Context, rch <-chan NetRpcResp, rid uint64) (NetRpcResp, error) {
select {
case resp := <-rch:
if resp.Type == NRpcErr {
var e NetRpcErr
if err := e.UnmarshalCBOR(bytes.NewReader(resp.Data)); err != nil {
return NetRpcResp{}, xerrors.Errorf("unmarshaling error data: %w", err)
}
var err error
switch e.Type {
case NRpcErrNotFound:
if e.Cid != nil {
err = ipld.ErrNotFound{
Cid: *e.Cid,
}
} else {
err = xerrors.Errorf("block not found, but cid was null")
}
case NRpcErrGeneric:
err = xerrors.Errorf("generic error")
default:
err = xerrors.Errorf("unknown error type")
}
return NetRpcResp{}, xerrors.Errorf("netstore error response: %s (%w)", e.Msg, err)
}
return resp, nil
case <-ctx.Done():
// todo send cancel req
n.respLk.Lock()
if n.respMap != nil {
delete(n.respMap, rid)
}
n.respLk.Unlock()
return NetRpcResp{}, ctx.Err()
}
}
func (n *NetworkStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
req, rch, err := n.sendRpc(NRpcHas, []cid.Cid{c}, nil)
if err != nil {
return false, err
}
resp, err := n.waitResp(ctx, rch, req)
if err != nil {
return false, err
}
if len(resp.Data) != 1 {
return false, xerrors.Errorf("expected reposnse length to be 1 byte")
}
switch resp.Data[0] {
case cbg.CborBoolTrue[0]:
return true, nil
case cbg.CborBoolFalse[0]:
return false, nil
default:
return false, xerrors.Errorf("has: bad response: %x", resp.Data[0])
}
}
func (n *NetworkStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
if err != nil {
return nil, err
}
resp, err := n.waitResp(ctx, rch, req)
if err != nil {
return nil, err
}
return blocks.NewBlockWithCid(resp.Data, c)
}
func (n *NetworkStore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
if err != nil {
return err
}
resp, err := n.waitResp(ctx, rch, req)
if err != nil {
return err
}
return callback(resp.Data) // todo return buf to pool
}
func (n *NetworkStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
req, rch, err := n.sendRpc(NRpcGetSize, []cid.Cid{c}, nil)
if err != nil {
return 0, err
}
resp, err := n.waitResp(ctx, rch, req)
if err != nil {
return 0, err
}
if len(resp.Data) != 4 {
return 0, xerrors.Errorf("expected getsize response to be 4 bytes, was %d", resp.Data)
}
return int(binary.LittleEndian.Uint32(resp.Data)), nil
}
func (n *NetworkStore) Put(ctx context.Context, block blocks.Block) error {
return n.PutMany(ctx, []blocks.Block{block})
}
func (n *NetworkStore) PutMany(ctx context.Context, blocks []blocks.Block) error {
// todo pool
cids := make([]cid.Cid, len(blocks))
blkDatas := make([][]byte, len(blocks))
for i, block := range blocks {
cids[i] = block.Cid()
blkDatas[i] = block.RawData()
}
req, rch, err := n.sendRpc(NRpcPut, cids, blkDatas)
if err != nil {
return err
}
_, err = n.waitResp(ctx, rch, req)
if err != nil {
return err
}
return nil
}
func (n *NetworkStore) DeleteBlock(ctx context.Context, c cid.Cid) error {
return n.DeleteMany(ctx, []cid.Cid{c})
}
func (n *NetworkStore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
req, rch, err := n.sendRpc(NRpcDelete, cids, nil)
if err != nil {
return err
}
_, err = n.waitResp(ctx, rch, req)
if err != nil {
return err
}
return nil
}
func (n *NetworkStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, xerrors.Errorf("not supported")
}
func (n *NetworkStore) HashOnRead(enabled bool) {
// todo
return
}
func (*NetworkStore) Flush(context.Context) error { return nil }
func (n *NetworkStore) Stop(ctx context.Context) error {
close(n.closing)
select {
case <-n.closed:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
var _ Blockstore = &NetworkStore{}
-237
View File
@@ -1,237 +0,0 @@
package blockstore
import (
"bytes"
"context"
"encoding/binary"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
block "github.com/ipfs/go-libipfs/blocks"
"github.com/libp2p/go-msgio"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
type NetworkStoreHandler struct {
msgStream msgio.ReadWriteCloser
bs Blockstore
}
// NOTE: This code isn't yet hardened to accept untrusted input. See TODOs here and in net.go
func HandleNetBstoreStream(ctx context.Context, bs Blockstore, mss msgio.ReadWriteCloser) *NetworkStoreHandler {
ns := &NetworkStoreHandler{
msgStream: mss,
bs: bs,
}
go ns.handle(ctx)
return ns
}
func (h *NetworkStoreHandler) handle(ctx context.Context) {
defer func() {
if err := h.msgStream.Close(); err != nil {
log.Errorw("error closing blockstore stream", "error", err)
}
}()
for {
var req NetRpcReq
ms, err := h.msgStream.ReadMsg()
if err != nil {
log.Warnw("bstore stream err", "error", err)
return
}
if err := req.UnmarshalCBOR(bytes.NewReader(ms)); err != nil {
return
}
h.msgStream.ReleaseMsg(ms)
switch req.Type {
case NRpcHas:
if len(req.Cid) != 1 {
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
res, err := h.bs.Has(ctx, req.Cid[0])
if err != nil {
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
var resData [1]byte
if res {
resData[0] = cbg.CborBoolTrue[0]
} else {
resData[0] = cbg.CborBoolFalse[0]
}
if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
log.Warnw("writing response", "error", err)
return
}
case NRpcGet:
if len(req.Cid) != 1 {
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
err := h.bs.View(ctx, req.Cid[0], func(bdata []byte) error {
return h.respond(req.ID, NRpcOK, bdata)
})
if err != nil {
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
case NRpcGetSize:
if len(req.Cid) != 1 {
if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
sz, err := h.bs.GetSize(ctx, req.Cid[0])
if err != nil {
if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
var resData [4]byte
binary.LittleEndian.PutUint32(resData[:], uint32(sz))
if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
log.Warnw("writing response", "error", err)
return
}
case NRpcPut:
blocks := make([]block.Block, len(req.Cid))
if len(req.Cid) != len(req.Data) {
if err := h.respondError(req.ID, xerrors.New("cid count didn't match data count"), cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
}
return
}
for i := range req.Cid {
blocks[i], err = block.NewBlockWithCid(req.Data[i], req.Cid[i])
if err != nil {
log.Warnw("make block", "error", err)
return
}
}
err := h.bs.PutMany(ctx, blocks)
if err != nil {
if err := h.respondError(req.ID, err, cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
log.Warnw("writing response", "error", err)
return
}
case NRpcDelete:
err := h.bs.DeleteMany(ctx, req.Cid)
if err != nil {
if err := h.respondError(req.ID, err, cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
log.Warnw("writing response", "error", err)
return
}
default:
if err := h.respondError(req.ID, xerrors.New("unsupported request type"), cid.Undef); err != nil {
log.Warnw("writing error response", "error", err)
return
}
continue
}
}
}
func (h *NetworkStoreHandler) respondError(req uint64, uerr error, c cid.Cid) error {
var resp NetRpcResp
resp.ID = req
resp.Type = NRpcErr
nerr := NetRpcErr{
Type: NRpcErrGeneric,
Msg: uerr.Error(),
}
if ipld.IsNotFound(uerr) {
nerr.Type = NRpcErrNotFound
nerr.Cid = &c
}
var edata bytes.Buffer
if err := nerr.MarshalCBOR(&edata); err != nil {
return xerrors.Errorf("marshaling error data: %w", err)
}
resp.Data = edata.Bytes()
var msg bytes.Buffer
if err := resp.MarshalCBOR(&msg); err != nil {
return xerrors.Errorf("marshaling error response: %w", err)
}
if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
return xerrors.Errorf("write error response: %w", err)
}
return nil
}
func (h *NetworkStoreHandler) respond(req uint64, rt NetRPCRespType, data []byte) error {
var resp NetRpcResp
resp.ID = req
resp.Type = rt
resp.Data = data
var msg bytes.Buffer
if err := resp.MarshalCBOR(&msg); err != nil {
return xerrors.Errorf("marshaling response: %w", err)
}
if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
return xerrors.Errorf("write response: %w", err)
}
return nil
}
-63
View File
@@ -1,63 +0,0 @@
package blockstore
import (
"context"
"fmt"
"io"
"testing"
ipld "github.com/ipfs/go-ipld-format"
block "github.com/ipfs/go-libipfs/blocks"
"github.com/libp2p/go-msgio"
"github.com/stretchr/testify/require"
)
func TestNetBstore(t *testing.T) {
ctx := context.Background()
cr, sw := io.Pipe()
sr, cw := io.Pipe()
cm := msgio.Combine(msgio.NewWriter(cw), msgio.NewReader(cr))
sm := msgio.Combine(msgio.NewWriter(sw), msgio.NewReader(sr))
bbs := NewMemorySync()
_ = HandleNetBstoreStream(ctx, bbs, sm)
nbs := NewNetworkStore(cm)
tb1 := block.NewBlock([]byte("aoeu"))
h, err := nbs.Has(ctx, tb1.Cid())
require.NoError(t, err)
require.False(t, h)
err = nbs.Put(ctx, tb1)
require.NoError(t, err)
h, err = nbs.Has(ctx, tb1.Cid())
require.NoError(t, err)
require.True(t, h)
sz, err := nbs.GetSize(ctx, tb1.Cid())
require.NoError(t, err)
require.Equal(t, 4, sz)
err = nbs.DeleteBlock(ctx, tb1.Cid())
require.NoError(t, err)
h, err = nbs.Has(ctx, tb1.Cid())
require.NoError(t, err)
require.False(t, h)
_, err = nbs.Get(ctx, tb1.Cid())
fmt.Println(err)
require.True(t, ipld.IsNotFound(err))
err = nbs.Put(ctx, tb1)
require.NoError(t, err)
b, err := nbs.Get(ctx, tb1.Cid())
require.NoError(t, err)
require.Equal(t, "aoeu", string(b.RawData()))
}
-100
View File
@@ -1,100 +0,0 @@
package blockstore
import (
"bytes"
"context"
"github.com/gorilla/websocket"
"github.com/libp2p/go-msgio"
"golang.org/x/xerrors"
)
type wsWrapper struct {
wc *websocket.Conn
nextMsg []byte
}
func (w *wsWrapper) Read(b []byte) (int, error) {
return 0, xerrors.New("read unsupported")
}
func (w *wsWrapper) ReadMsg() ([]byte, error) {
if w.nextMsg != nil {
nm := w.nextMsg
w.nextMsg = nil
return nm, nil
}
mt, r, err := w.wc.NextReader()
if err != nil {
return nil, err
}
switch mt {
case websocket.BinaryMessage, websocket.TextMessage:
default:
return nil, xerrors.Errorf("unexpected message type")
}
// todo pool
// todo limit sizes
var mbuf bytes.Buffer
if _, err := mbuf.ReadFrom(r); err != nil {
return nil, err
}
return mbuf.Bytes(), nil
}
func (w *wsWrapper) ReleaseMsg(bytes []byte) {
// todo use a pool
}
func (w *wsWrapper) NextMsgLen() (int, error) {
if w.nextMsg != nil {
return len(w.nextMsg), nil
}
mt, msg, err := w.wc.ReadMessage()
if err != nil {
return 0, err
}
switch mt {
case websocket.BinaryMessage, websocket.TextMessage:
default:
return 0, xerrors.Errorf("unexpected message type")
}
w.nextMsg = msg
return len(w.nextMsg), nil
}
func (w *wsWrapper) Write(bytes []byte) (int, error) {
return 0, xerrors.New("write unsupported")
}
func (w *wsWrapper) WriteMsg(bytes []byte) error {
return w.wc.WriteMessage(websocket.BinaryMessage, bytes)
}
func (w *wsWrapper) Close() error {
return w.wc.Close()
}
var _ msgio.ReadWriteCloser = &wsWrapper{}
func wsConnToMio(wc *websocket.Conn) msgio.ReadWriteCloser {
return &wsWrapper{
wc: wc,
}
}
func HandleNetBstoreWS(ctx context.Context, bs Blockstore, wc *websocket.Conn) *NetworkStoreHandler {
return HandleNetBstoreStream(ctx, bs, wsConnToMio(wc))
}
func NewNetworkStoreWS(wc *websocket.Conn) *NetworkStore {
return NewNetworkStore(wsConnToMio(wc))
}
+3 -2
View File
@@ -5,9 +5,10 @@ import (
"io"
"os"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)
type Checkpoint struct {
+2 -1
View File
@@ -5,8 +5,9 @@ import (
"io"
"os"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type ColdSetWriter struct {
+3 -2
View File
@@ -12,10 +12,11 @@ import (
"sync"
"time"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"go.uber.org/multierr"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
type debugLog struct {
+2 -1
View File
@@ -3,8 +3,9 @@ package splitstore
import (
"errors"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
var errMarkSetClosed = errors.New("markset closed")
+6 -6
View File
@@ -6,13 +6,13 @@ import (
"runtime"
"sync"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"github.com/ipfs/go-cid"
"go.uber.org/zap"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/system"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
"go.uber.org/zap"
cid "github.com/ipfs/go-cid"
)
type BadgerMarkSetEnv struct {
@@ -351,7 +351,7 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
persist := s.persist
s.mx.RUnlock()
if persist && !system.BadgerFsyncDisable { // WARNING: disabling sync makes recovery from crash during critical section unsound
if persist {
return s.db.Sync()
}
+2 -1
View File
@@ -7,8 +7,9 @@ import (
"path/filepath"
"sync"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type MapMarkSetEnv struct {
+2 -2
View File
@@ -1,10 +1,10 @@
// stm: #unit
//stm: #unit
package splitstore
import (
"testing"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
+34 -171
View File
@@ -8,22 +8,22 @@ import (
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/stats"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/metrics"
"go.opencensus.io/stats"
)
var (
@@ -43,16 +43,8 @@ var (
// compactionIndexKey stores the compaction index (serial number)
compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
// stores the prune index (serial number)
pruneIndexKey = dstore.NewKey("/splitstore/pruneIndex")
// stores the base epoch of last prune in the metadata store
pruneEpochKey = dstore.NewKey("/splitstore/pruneEpoch")
log = logging.Logger("splitstore")
errClosing = errors.New("splitstore is closing")
// set this to true if you are debugging the splitstore to enable debug logging
enableDebugLog = false
// set this to true if you want to track origin stack traces in the write log
@@ -62,16 +54,6 @@ var (
upgradeBoundary = build.Finality
)
type CompactType int
const (
none CompactType = iota
warmup
hot
cold
check
)
func init() {
if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
enableDebugLog = true
@@ -98,10 +80,6 @@ type Config struct {
// and directly purges cold blocks.
DiscardColdBlocks bool
// UniversalColdBlocks indicates whether all blocks being garbage collected and purged
// from the hotstore should be written to the cold store
UniversalColdBlocks bool
// HotstoreMessageRetention indicates the hotstore retention policy for messages.
// It has the following semantics:
// - a value of 0 will only retain messages within the compaction boundary (4 finalities)
@@ -115,23 +93,6 @@ type Config struct {
// A positive value is the number of compactions before a full GC is performed;
// a value of 1 will perform full GC in every compaction.
HotStoreFullGCFrequency uint64
// HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take.
// This is not a hard limit, it is possible for the hotstore to exceed the target
// for example if state grows massively between compactions. The splitstore
// will make a best effort to avoid overflowing the target and in practice should
// never overflow. This field is used when doing GC at the end of a compaction to
// adaptively choose moving GC
HotstoreMaxSpaceTarget uint64
// Moving GC will be triggered when total moving size exceeds
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
HotstoreMaxSpaceThreshold uint64
// Safety buffer to prevent moving GC from overflowing disk.
// Moving GC will not occur when total moving size exceeds
// HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
HotstoreMaxSpaceSafetyBuffer uint64
}
// ChainAccessor allows the Splitstore to access the chain. It will most likely
@@ -156,9 +117,8 @@ type hotstore interface {
}
type SplitStore struct {
compacting int32 // flag for when compaction is in progress
compactType CompactType // compaction type, protected by compacting atomic, only meaningful when compacting == 1
closing int32 // the splitstore is closing
compacting int32 // compaction/prune/warmup in progress
closing int32 // the splitstore is closing
cfg *Config
path string
@@ -166,7 +126,6 @@ type SplitStore struct {
mx sync.Mutex
warmupEpoch abi.ChainEpoch // protected by mx
baseEpoch abi.ChainEpoch // protected by compaction lock
pruneEpoch abi.ChainEpoch // protected by compaction lock
headChangeMx sync.Mutex
@@ -181,17 +140,10 @@ type SplitStore struct {
markSetSize int64
compactionIndex int64
pruneIndex int64
onlineGCCnt int64
ctx context.Context
cancel func()
outOfSync int32 // for fast checking
chainSyncMx sync.Mutex
chainSyncCond sync.Cond
chainSyncFinished bool // protected by chainSyncMx
debug *debugLog
// transactional protection for concurrent read/writes during compaction
@@ -218,17 +170,6 @@ type SplitStore struct {
// registered protectors
protectors []func(func(cid.Cid) error) error
// dag sizes measured during latest compaction
// logged and used for GC strategy
// protected by compaction lock
szWalk int64
szProtectedTxns int64
szKeys int64 // approximate, not counting keys protected when entering critical section
// protected by txnLk
szMarkedLiveRefs int64
}
var _ bstore.Blockstore = (*SplitStore)(nil)
@@ -266,7 +207,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
ss.txnViewsCond.L = &ss.txnViewsMx
ss.txnSyncCond.L = &ss.txnSyncMx
ss.chainSyncCond.L = &ss.chainSyncMx
ss.ctx, ss.cancel = context.WithCancel(context.Background())
ss.reifyCond.L = &ss.reifyMx
@@ -287,13 +227,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
return nil, xerrors.Errorf("error resuming compaction: %w", err)
}
}
if ss.pruneCheckpointExists() {
log.Info("found prune checkpoint; resuming prune")
if err := ss.completePrune(); err != nil {
markSetEnv.Close() //nolint:errcheck
return nil, xerrors.Errorf("error resuming prune: %w", err)
}
}
return ss, nil
}
@@ -327,14 +260,8 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
if has {
return s.has(cid)
}
switch s.compactType {
case hot:
return s.cold.Has(ctx, cid)
case cold:
return s.hot.Has(ctx, cid)
default:
return false, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
}
return s.cold.Has(ctx, cid)
}
has, err := s.hot.Has(ctx, cid)
@@ -349,11 +276,8 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
}
has, err = s.cold.Has(ctx, cid)
if has {
s.trackTxnRef(cid)
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
if has && bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
return has, err
@@ -383,31 +307,24 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
if has {
return s.get(cid)
}
switch s.compactType {
case hot:
return s.cold.Get(ctx, cid)
case cold:
return s.hot.Get(ctx, cid)
default:
return nil, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
}
return s.cold.Get(ctx, cid)
}
blk, err := s.hot.Get(ctx, cid)
switch {
case err == nil:
switch err {
case nil:
s.trackTxnRef(cid)
return blk, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
blk, err = s.cold.Get(ctx, cid)
if err == nil {
s.trackTxnRef(cid)
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
@@ -444,31 +361,24 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
if has {
return s.getSize(cid)
}
switch s.compactType {
case hot:
return s.cold.GetSize(ctx, cid)
case cold:
return s.hot.GetSize(ctx, cid)
default:
return 0, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
}
return s.cold.GetSize(ctx, cid)
}
size, err := s.hot.GetSize(ctx, cid)
switch {
case err == nil:
switch err {
case nil:
s.trackTxnRef(cid)
return size, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
size, err = s.cold.GetSize(ctx, cid)
if err == nil {
s.trackTxnRef(cid)
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
@@ -482,23 +392,6 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
}
}
func (s *SplitStore) Flush(ctx context.Context) error {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
if err := s.cold.Flush(ctx); err != nil {
return err
}
if err := s.hot.Flush(ctx); err != nil {
return err
}
if err := s.ds.Sync(ctx, dstore.Key{}); err != nil {
return err
}
return nil
}
func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
if isIdentiyCid(blk.Cid()) {
return nil
@@ -515,12 +408,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
s.debug.LogWrite(blk)
// critical section
if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
if s.txnMarkSet != nil {
s.markLiveRefs([]cid.Cid{blk.Cid()})
return nil
}
s.trackTxnRef(blk.Cid())
s.trackTxnRef(blk.Cid())
return nil
}
@@ -566,12 +459,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
s.debug.LogWriteMany(blks)
// critical section
if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
if s.txnMarkSet != nil {
s.markLiveRefs(batch)
return nil
}
s.trackTxnRefMany(batch)
s.trackTxnRefMany(batch)
return nil
}
@@ -643,14 +536,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
if has {
return s.view(cid, cb)
}
switch s.compactType {
case hot:
return s.cold.View(ctx, cid, cb)
case cold:
return s.hot.View(ctx, cid, cb)
default:
return xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
}
return s.cold.View(ctx, cid, cb)
}
// views are (optimistically) protected two-fold:
@@ -665,7 +552,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
defer s.viewDone()
err := s.hot.View(ctx, cid, cb)
if ipld.IsNotFound(err) {
switch err {
case bstore.ErrNotFound:
if s.isWarm() {
s.debug.LogReadMiss(cid)
}
@@ -679,8 +567,10 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return err
default:
return err
}
return err
}
func (s *SplitStore) isWarm() bool {
@@ -736,23 +626,6 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
return xerrors.Errorf("error loading base epoch: %w", err)
}
// load prune epoch from metadata ds
bs, err = s.ds.Get(s.ctx, pruneEpochKey)
switch err {
case nil:
s.pruneEpoch = bytesToEpoch(bs)
case dstore.ErrNotFound:
if curTs == nil {
//this can happen in some tests
break
}
if err := s.setPruneEpoch(curTs.Height()); err != nil {
return xerrors.Errorf("error saving prune epoch: %w", err)
}
default:
return xerrors.Errorf("error loading prune epoch: %w", err)
}
// load warmup epoch from metadata ds
bs, err = s.ds.Get(s.ctx, warmupEpochKey)
switch err {
@@ -828,11 +701,6 @@ func (s *SplitStore) Close() error {
s.txnSyncCond.Broadcast()
s.txnSyncMx.Unlock()
s.chainSyncMx.Lock()
s.chainSyncFinished = true
s.chainSyncCond.Broadcast()
s.chainSyncMx.Unlock()
log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
for atomic.LoadInt32(&s.compacting) == 1 {
time.Sleep(time.Second)
@@ -857,8 +725,3 @@ func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
s.baseEpoch = epoch
return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
}
func (s *SplitStore) setPruneEpoch(epoch abi.ChainEpoch) error {
s.pruneEpoch = epoch
return s.ds.Put(s.ctx, pruneEpochKey, epochToBytes(epoch))
}
+5 -7
View File
@@ -8,9 +8,10 @@ import (
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -25,7 +26,6 @@ func (s *SplitStore) Check() error {
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
}
s.compactType = check
if s.compactionIndex == 0 {
atomic.StoreInt32(&s.compacting, 0)
@@ -95,7 +95,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
defer visitor.Close() //nolint
size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
@@ -125,7 +125,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
return nil
}, func(cid.Cid) error { return nil })
})
if err != nil {
err = xerrors.Errorf("error walking chain: %w", err)
@@ -133,7 +133,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return err
}
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size)
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
write("--")
write("cold: %d missing: %d", *coldCnt, *missingCnt)
write("DONE")
@@ -147,8 +147,6 @@ func (s *SplitStore) Info() map[string]interface{} {
info["base epoch"] = s.baseEpoch
info["warmup epoch"] = s.warmupEpoch
info["compactions"] = s.compactionIndex
info["prunes"] = s.pruneIndex
info["compacting"] = s.compacting == 1
sizer, ok := s.hot.(bstore.BlockstoreSize)
if ok {
+115 -340
View File
@@ -10,19 +10,20 @@ import (
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
cbg "github.com/whyrusleeping/cbor-gen"
"go.opencensus.io/stats"
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/metrics"
"go.opencensus.io/stats"
)
var (
@@ -52,12 +53,6 @@ var (
// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
// we have synced.
SyncWaitTime = 30 * time.Second
// This is a testing flag that should always be true when running a node. itests rely on the rough hack
// of starting genesis so far in the past that they exercise catchup mining to mine
// blocks quickly and so disabling syncgap checking is necessary to test compaction
// without a deep structural improvement of itests.
CheckSyncGap = true
)
var (
@@ -66,8 +61,7 @@ var (
)
const (
batchSize = 16384
cidKeySize = 128
batchSize = 16384
)
func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -88,38 +82,10 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
// this is guaranteed by the chainstore, and it is pervasive in all lotus
// -- if that ever changes then all hell will break loose in general and
// we will have a rance to protectTipSets here.
// Regardless, we put a mutex in HeadChange just to be safe
// Reagrdless, we put a mutex in HeadChange just to be safe
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
// we are currently compacting
// 1. Signal sync condition to yield compaction when out of sync and resume when in sync
timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
/* Chain out of sync */
if atomic.CompareAndSwapInt32(&s.outOfSync, 0, 1) {
// transition from in sync to out of sync
s.chainSyncMx.Lock()
s.chainSyncFinished = false
s.chainSyncMx.Unlock()
}
// already out of sync, no signaling necessary
}
// TODO: ok to use hysteresis with no transitions between 30s and 1m?
if time.Since(timestamp) < SyncWaitTime {
/* Chain in sync */
if atomic.CompareAndSwapInt32(&s.outOfSync, 0, 0) {
// already in sync, no signaling necessary
} else {
// transition from out of sync to in sync
s.chainSyncMx.Lock()
s.chainSyncFinished = true
s.chainSyncCond.Broadcast()
s.chainSyncMx.Unlock()
}
}
// 2. protect the new tipset(s)
// we are currently compacting -- protect the new tipset(s)
s.protectTipSets(apply)
return nil
}
@@ -131,8 +97,7 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
}
timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
if time.Since(timestamp) > SyncGapTime {
// don't attempt compaction before we have caught up syncing
atomic.StoreInt32(&s.compacting, 0)
return nil
@@ -147,7 +112,6 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
if epoch-s.baseEpoch > CompactionThreshold {
// it's time to compact -- prepare the transaction and go!
s.beginTxnProtect()
s.compactType = hot
go func() {
defer atomic.StoreInt32(&s.compacting, 0)
defer s.endTxnProtect()
@@ -159,7 +123,6 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
log.Infow("compaction done", "took", time.Since(start))
}()
// only prune if auto prune is enabled and after at least one compaction
} else {
// no compaction necessary
atomic.StoreInt32(&s.compacting, 0)
@@ -203,8 +166,6 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
doSync := time.Since(timestamp) < SyncWaitTime
go func() {
// we are holding the txnLk while marking
// so critical section cannot delete
if doSync {
defer func() {
s.txnSyncMx.Lock()
@@ -228,11 +189,9 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
log.Debugf("marking %d live refs", len(cids))
startMark := time.Now()
szMarked := new(int64)
count := new(int32)
visitor := newConcurrentVisitor()
walkObject := func(c cid.Cid) (int64, error) {
walkObject := func(c cid.Cid) error {
return s.walkObjectIncomplete(c, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
@@ -259,12 +218,10 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
// optimize the common case of single put
if len(cids) == 1 {
sz, err := walkObject(cids[0])
if err != nil {
if err := walkObject(cids[0]); err != nil {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
atomic.AddInt64(szMarked, sz)
return
}
@@ -276,11 +233,9 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
worker := func() error {
for c := range workch {
sz, err := walkObject(c)
if err != nil {
if err := walkObject(c); err != nil {
return err
}
atomic.AddInt64(szMarked, sz)
}
return nil
@@ -303,8 +258,7 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count, "size marked", *szMarked)
s.szMarkedLiveRefs += atomic.LoadInt64(szMarked)
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
}
// transactionally protect a view
@@ -397,7 +351,6 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
log.Infow("protecting transactional references", "refs", len(txnRefs))
count := 0
sz := new(int64)
workch := make(chan cid.Cid, len(txnRefs))
startProtect := time.Now()
@@ -430,11 +383,10 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
worker := func() error {
for c := range workch {
szTxn, err := s.doTxnProtect(c, markSet)
err := s.doTxnProtect(c, markSet)
if err != nil {
return xerrors.Errorf("error protecting transactional references to %s: %w", c, err)
}
atomic.AddInt64(sz, szTxn)
}
return nil
}
@@ -447,16 +399,16 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
if err := g.Wait(); err != nil {
return err
}
s.szProtectedTxns += atomic.LoadInt64(sz)
log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count, "protected size", sz)
log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count)
}
}
// transactionally protect a reference by walking the object and marking.
// concurrent markings are short circuited by checking the markset.
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error) {
if err := s.checkYield(); err != nil {
return 0, err
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
if err := s.checkClosing(); err != nil {
return err
}
// Note: cold objects are deleted heaviest first, so the consituents of an object
@@ -480,7 +432,7 @@ func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error)
},
func(c cid.Cid) error {
if s.txnMissing != nil {
log.Debugf("missing object reference %s in %s", c, root)
log.Warnf("missing object reference %s in %s", c, root)
s.txnRefsMx.Lock()
s.txnMissing[c] = struct{}{}
s.txnRefsMx.Unlock()
@@ -523,7 +475,6 @@ func (s *SplitStore) applyProtectors() error {
// - At this point we are ready to begin purging:
// - We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
// - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
//
// - We then end the transaction and compact/gc the hotstore.
func (s *SplitStore) compact(curTs *types.TipSet) {
log.Info("waiting for active views to complete")
@@ -547,7 +498,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
// might be potentially inconsistent; abort compaction and notify the user to intervene.
return xerrors.Errorf("checkpoint exists; aborting compaction")
}
s.clearSizeMeasurements()
currentEpoch := curTs.Height()
boundaryEpoch := currentEpoch - CompactionBoundary
@@ -567,13 +517,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
defer markSet.Close() //nolint:errcheck
defer s.debug.Flush()
coldSet, err := s.markSetEnv.New("cold", s.markSetSize)
if err != nil {
return xerrors.Errorf("error creating cold mark set: %w", err)
}
defer coldSet.Close() //nolint:errcheck
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -591,52 +535,25 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
startMark := time.Now()
count := new(int64)
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
coldCount := new(int64)
fCold := func(c cid.Cid) error {
// Writes to cold set optimized away in universal and discard mode
//
// Nothing gets written to cold store in discard mode so no cold objects to write
// Everything not marked hot gets written to cold store in universal mode so no need to track cold objects separately
if s.cfg.DiscardColdBlocks || s.cfg.UniversalColdBlocks {
visit, err := markSet.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return errStopWalk
}
atomic.AddInt64(count, 1)
return nil
}
})
if isUnitaryObject(c) {
return errStopWalk
}
visit, err := coldSet.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return errStopWalk
}
atomic.AddInt64(coldCount, 1)
return nil
}
fHot := func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
visit, err := markSet.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return errStopWalk
}
atomic.AddInt64(count, 1)
return nil
}
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{}, fHot, fCold)
if err != nil {
return xerrors.Errorf("error marking: %w", err)
}
@@ -645,7 +562,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
log.Infow("marking done", "took", time.Since(startMark), "marked", *count)
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -655,7 +572,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return xerrors.Errorf("error protecting transactional refs: %w", err)
}
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -669,14 +586,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
defer coldw.Close() //nolint:errcheck
purgew, err := NewColdSetWriter(s.discardSetPath())
if err != nil {
return xerrors.Errorf("error creating deadset: %w", err)
}
defer purgew.Close() //nolint:errcheck
// some stats for logging
var hotCnt, coldCnt, purgeCnt int64
var hotCnt, coldCnt int
err = s.hot.ForEachKey(func(c cid.Cid) error {
// was it marked?
mark, err := markSet.Has(c)
@@ -689,50 +601,30 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return nil
}
// it needs to be removed from hot store, mark it as candidate for purge
if err := purgew.Write(c); err != nil {
return xerrors.Errorf("error writing cid to purge set: %w", err)
}
purgeCnt++
coldMark, err := coldSet.Has(c)
if err != nil {
return xerrors.Errorf("error checking cold mark set for %s: %w", c, err)
}
// Discard mode: coldMark == false, s.cfg.UniversalColdBlocks == false, always return here, no writes to cold store
// Universal mode: coldMark == false, s.cfg.UniversalColdBlocks == true, never stop here, all writes to cold store
// Otherwise: s.cfg.UniversalColdBlocks == false, if !coldMark stop here and don't write to cold store, if coldMark continue and write to cold store
if !coldMark && !s.cfg.UniversalColdBlocks { // universal mode means mark everything as cold
return nil
}
// it's cold, mark as candidate for move
// it's cold, mark it as candidate for move
if err := coldw.Write(c); err != nil {
return xerrors.Errorf("error writing cid to cold set")
return xerrors.Errorf("error writing cid to coldstore: %w", err)
}
coldCnt++
return nil
})
if err != nil {
return xerrors.Errorf("error collecting cold objects: %w", err)
}
if err := purgew.Close(); err != nil {
return xerrors.Errorf("erroring closing purgeset: %w", err)
}
if err := coldw.Close(); err != nil {
return xerrors.Errorf("error closing coldset: %w", err)
}
log.Infow("cold collection done", "took", time.Since(startCollect))
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "purge", purgeCnt)
s.szKeys = hotCnt * cidKeySize
stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(hotCnt))
stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(coldCnt))
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -741,7 +633,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
// possibly delete objects we didn't have when we were collecting cold objects)
s.waitForMissingRefs(markSet)
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -761,7 +653,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
log.Infow("moving done", "took", time.Since(startMove))
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -770,12 +662,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
}
purger, err := NewColdSetReader(s.discardSetPath())
if err != nil {
return xerrors.Errorf("error opening coldset: %w", err)
}
defer purger.Close() //nolint:errcheck
// 4. Purge cold objects with checkpointing for recovery.
// This is the critical section of compaction, whereby any cold object not in the markSet is
// considered already deleted.
@@ -792,7 +678,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
// wait for the head to catch up so that the current tipset is marked
s.waitForTxnSync()
s.waitForSync()
if err := s.checkClosing(); err != nil {
return err
@@ -807,13 +693,13 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
// 5. purge cold objects from the hotstore, taking protected references into account
log.Info("purging cold objects from the hotstore")
startPurge := time.Now()
err = s.purge(purger, checkpoint, markSet)
err = s.purge(coldr, checkpoint, markSet)
if err != nil {
return xerrors.Errorf("error purging cold objects: %w", err)
}
log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
s.endCriticalSection()
log.Infow("critical section done", "total protected size", s.szProtectedTxns, "total marked live size", s.szMarkedLiveRefs)
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
@@ -827,13 +713,10 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
if err := os.Remove(s.coldSetPath()); err != nil {
log.Warnf("error removing coldset: %s", err)
}
if err := os.Remove(s.discardSetPath()); err != nil {
log.Warnf("error removing discardset: %s", err)
}
// we are done; do some housekeeping
s.endTxnProtect()
s.gcHotAfterCompaction()
s.gcHotstore()
err = s.setBaseEpoch(boundaryEpoch)
if err != nil {
@@ -893,12 +776,8 @@ func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
return nil
}
func (s *SplitStore) waitForTxnSync() {
func (s *SplitStore) waitForSync() {
log.Info("waiting for sync")
if !CheckSyncGap {
log.Warnf("If you see this outside of test it is a serious splitstore issue")
return
}
startWait := time.Now()
defer func() {
log.Infow("waiting for sync done", "took", time.Since(startWait))
@@ -912,25 +791,6 @@ func (s *SplitStore) waitForTxnSync() {
}
}
// Block compaction operations if chain sync has fallen behind
func (s *SplitStore) waitForSync() {
if atomic.LoadInt32(&s.outOfSync) == 0 {
return
}
s.chainSyncMx.Lock()
defer s.chainSyncMx.Unlock()
for !s.chainSyncFinished {
s.chainSyncCond.Wait()
}
}
// Combined sync and closing check
func (s *SplitStore) checkYield() error {
s.waitForSync()
return s.checkClosing()
}
func (s *SplitStore) endTxnProtect() {
s.txnLk.Lock()
defer s.txnLk.Unlock()
@@ -957,7 +817,7 @@ func (s *SplitStore) endCriticalSection() {
}
func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
visitor ObjectVisitor, fHot, fCold func(cid.Cid) error) error {
visitor ObjectVisitor, f func(cid.Cid) error) error {
var walked ObjectVisitor
var mx sync.Mutex
// we copy the tipset first into a new slice, which allows us to reuse it in every epoch.
@@ -965,11 +825,6 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
copy(toWalk, ts.Cids())
walkCnt := new(int64)
scanCnt := new(int64)
szWalk := new(int64)
tsRef := func(blkCids []cid.Cid) (cid.Cid, error) {
return types.NewTipSetKey(blkCids...).Cid()
}
stopWalk := func(_ cid.Cid) error { return errStopWalk }
@@ -984,7 +839,7 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
atomic.AddInt64(walkCnt, 1)
if err := fHot(c); err != nil {
if err := f(c); err != nil {
return err
}
@@ -992,73 +847,39 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
err = s.view(c, func(data []byte) error {
return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
})
if err != nil {
return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
}
// tipset CID references are retained
pRef, err := tsRef(hdr.Parents)
if err != nil {
return xerrors.Errorf("error computing cid reference to parent tipset")
}
sz, err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking parent tipset cid reference")
}
atomic.AddInt64(szWalk, sz)
// message are retained if within the inclMsgs boundary
if hdr.Height >= inclMsgs && hdr.Height > 0 {
if inclMsgs < inclState {
// we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we
// synced from snapshot and have a long HotStoreMessageRetentionPolicy.
sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk)
if err != nil {
if err := s.walkObjectIncomplete(hdr.Messages, visitor, f, stopWalk); err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
atomic.AddInt64(szWalk, sz)
sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk)
if err != nil {
if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, f, stopWalk); err != nil {
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
} else {
sz, err = s.walkObject(hdr.Messages, visitor, fHot)
if err != nil {
if err := s.walkObject(hdr.Messages, visitor, f); err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
atomic.AddInt64(szWalk, sz)
sz, err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk)
if err != nil {
if err := s.walkObject(hdr.ParentMessageReceipts, visitor, f); err != nil {
return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
}
}
// messages and receipts outside of inclMsgs are included in the cold store
if hdr.Height < inclMsgs && hdr.Height > 0 {
sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
}
atomic.AddInt64(szWalk, sz)
sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk)
if err != nil {
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
atomic.AddInt64(szWalk, sz)
}
// state is only retained if within the inclState boundary, with the exception of genesis
if hdr.Height >= inclState || hdr.Height == 0 {
sz, err := s.walkObject(hdr.ParentStateRoot, visitor, fHot)
if err != nil {
if err := s.walkObject(hdr.ParentStateRoot, visitor, f); err != nil {
return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
}
atomic.AddInt64(szWalk, sz)
atomic.AddInt64(scanCnt, 1)
}
@@ -1071,20 +892,9 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
return nil
}
// retain ref to chain head
hRef, err := tsRef(ts.Cids())
if err != nil {
return xerrors.Errorf("error computing cid reference to parent tipset")
}
sz, err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk)
if err != nil {
return xerrors.Errorf("error walking parent tipset cid reference")
}
atomic.AddInt64(szWalk, sz)
for len(toWalk) > 0 {
// walking can take a while, so check this with every opportunity
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
@@ -1124,129 +934,123 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
}
}
log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt, "walk size", szWalk)
s.szWalk = atomic.LoadInt64(szWalk)
log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt)
return nil
}
func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) (int64, error) {
var sz int64
func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error {
visit, err := visitor.Visit(c)
if err != nil {
return 0, xerrors.Errorf("error visiting object: %w", err)
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return sz, nil
return nil
}
if err := f(c); err != nil {
if err == errStopWalk {
return sz, nil
return nil
}
return 0, err
return err
}
if c.Prefix().Codec != cid.DagCBOR {
return sz, nil
return nil
}
// check this before recursing
if err := s.checkYield(); err != nil {
return 0, err
if err := s.checkClosing(); err != nil {
return err
}
var links []cid.Cid
err = s.view(c, func(data []byte) error {
sz += int64(len(data))
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
}
for _, c := range links {
szLink, err := s.walkObject(c, visitor, f)
err := s.walkObject(c, visitor, f)
if err != nil {
return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
}
sz += szLink
}
return sz, nil
return nil
}
// like walkObject, but the object may be potentially incomplete (references missing)
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) (int64, error) {
var sz int64
func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error {
visit, err := visitor.Visit(c)
if err != nil {
return 0, xerrors.Errorf("error visiting object: %w", err)
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return sz, nil
return nil
}
// occurs check -- only for DAGs
if c.Prefix().Codec == cid.DagCBOR {
has, err := s.has(c)
if err != nil {
return 0, xerrors.Errorf("error occur checking %s: %w", c, err)
return xerrors.Errorf("error occur checking %s: %w", c, err)
}
if !has {
err = missing(c)
if err == errStopWalk {
return sz, nil
return nil
}
return 0, err
return err
}
}
if err := f(c); err != nil {
if err == errStopWalk {
return sz, nil
return nil
}
return 0, err
return err
}
if c.Prefix().Codec != cid.DagCBOR {
return sz, nil
return nil
}
// check this before recursing
if err := s.checkYield(); err != nil {
return sz, err
if err := s.checkClosing(); err != nil {
return err
}
var links []cid.Cid
err = s.view(c, func(data []byte) error {
sz += int64(len(data))
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
}
for _, c := range links {
szLink, err := s.walkObjectIncomplete(c, visitor, f, missing)
err := s.walkObjectIncomplete(c, visitor, f, missing)
if err != nil {
return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
}
sz += szLink
}
return sz, nil
return nil
}
// internal version used during compaction and related operations
@@ -1261,10 +1065,13 @@ func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
}
err := s.hot.View(s.ctx, c, cb)
if ipld.IsNotFound(err) {
switch err {
case bstore.ErrNotFound:
return s.cold.View(s.ctx, c, cb)
default:
return err
}
return err
}
func (s *SplitStore) has(c cid.Cid) (bool, error) {
@@ -1283,10 +1090,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
blk, err := s.hot.Get(s.ctx, c)
switch {
case err == nil:
switch err {
case nil:
return blk, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
return s.cold.Get(s.ctx, c)
default:
return nil, err
@@ -1295,10 +1102,10 @@ func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
func (s *SplitStore) getSize(c cid.Cid) (int, error) {
sz, err := s.hot.GetSize(s.ctx, c)
switch {
case err == nil:
switch err {
case nil:
return sz, nil
case ipld.IsNotFound(err):
case bstore.ErrNotFound:
return s.cold.GetSize(s.ctx, c)
default:
return 0, err
@@ -1309,12 +1116,13 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
batch := make([]blocks.Block, 0, batchSize)
err := coldr.ForEach(func(c cid.Cid) error {
if err := s.checkYield(); err != nil {
if err := s.checkClosing(); err != nil {
return err
}
blk, err := s.hot.Get(s.ctx, c)
if err != nil {
if ipld.IsNotFound(err) {
if err == bstore.ErrNotFound {
log.Warnf("hotstore missing block %s", c)
return nil
}
@@ -1329,7 +1137,6 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
return xerrors.Errorf("error putting batch to coldstore: %w", err)
}
batch = batch[:0]
}
return nil
@@ -1418,17 +1225,8 @@ func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoin
return 0, liveCnt, nil
}
switch s.compactType {
case hot:
if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
}
case cold:
if err := s.cold.DeleteMany(s.ctx, deadCids); err != nil {
return 0, liveCnt, xerrors.Errorf("error purging dead objects: %w", err)
}
default:
return 0, liveCnt, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
}
s.debug.LogDelete(deadCids)
@@ -1445,28 +1243,15 @@ func (s *SplitStore) coldSetPath() string {
return filepath.Join(s.path, "coldset")
}
func (s *SplitStore) discardSetPath() string {
return filepath.Join(s.path, "deadset")
}
func (s *SplitStore) checkpointPath() string {
return filepath.Join(s.path, "checkpoint")
}
func (s *SplitStore) pruneCheckpointPath() string {
return filepath.Join(s.path, "prune-checkpoint")
}
func (s *SplitStore) checkpointExists() bool {
_, err := os.Stat(s.checkpointPath())
return err == nil
}
func (s *SplitStore) pruneCheckpointExists() bool {
_, err := os.Stat(s.pruneCheckpointPath())
return err == nil
}
func (s *SplitStore) completeCompaction() error {
checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
if err != nil {
@@ -1487,7 +1272,6 @@ func (s *SplitStore) completeCompaction() error {
defer markSet.Close() //nolint:errcheck
// PURGE
s.compactType = hot
log.Info("purging cold objects from the hotstore")
startPurge := time.Now()
err = s.completePurge(coldr, checkpoint, last, markSet)
@@ -1510,11 +1294,9 @@ func (s *SplitStore) completeCompaction() error {
if err := os.Remove(s.coldSetPath()); err != nil {
log.Warnf("error removing coldset: %s", err)
}
s.compactType = none
// Note: at this point we can start the splitstore; base epoch is not
// incremented here so a compaction should run on the first head
// change, which will trigger gc on the hotstore.
// Note: at this point we can start the splitstore; a compaction should run on
// the first head change, which will trigger gc on the hotstore.
// We don't mind the second (back-to-back) compaction as the head will
// have advanced during marking and coldset accumulation.
return nil
@@ -1572,13 +1354,6 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint,
return nil
}
func (s *SplitStore) clearSizeMeasurements() {
s.szKeys = 0
s.szMarkedLiveRefs = 0
s.szProtectedTxns = 0
s.szWalk = 0
}
// I really don't like having this code, but we seem to have some occasional DAG references with
// missing constituents. During testing in mainnet *some* of these references *sometimes* appeared
// after a little bit.
@@ -1619,7 +1394,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
missing = make(map[cid.Cid]struct{})
for c := range towalk {
_, err := s.walkObjectIncomplete(c, visitor,
err := s.walkObjectIncomplete(c, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
+15 -15
View File
@@ -4,9 +4,8 @@ import (
"context"
"errors"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
bstore "github.com/filecoin-project/lotus/blockstore"
)
@@ -43,7 +42,6 @@ func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
}
func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
if err != nil {
@@ -54,10 +52,12 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
}
blk, err := es.s.hot.Get(ctx, c)
if ipld.IsNotFound(err) {
switch err {
case bstore.ErrNotFound:
return es.s.cold.Get(ctx, c)
default:
return blk, err
}
return blk, err
}
func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
@@ -71,14 +71,12 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
}
size, err := es.s.hot.GetSize(ctx, c)
if ipld.IsNotFound(err) {
switch err {
case bstore.ErrNotFound:
return es.s.cold.GetSize(ctx, c)
default:
return size, err
}
return size, err
}
func (es *exposedSplitStore) Flush(ctx context.Context) error {
return es.s.Flush(ctx)
}
func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
@@ -106,9 +104,11 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
}
err := es.s.hot.View(ctx, c, f)
if ipld.IsNotFound(err) {
switch err {
case bstore.ErrNotFound:
return es.s.cold.View(ctx, c, f)
}
return err
default:
return err
}
}
+4 -71
View File
@@ -7,96 +7,29 @@ import (
bstore "github.com/filecoin-project/lotus/blockstore"
)
const (
// Fraction of garbage in badger vlog for online GC traversal to collect garbage
AggressiveOnlineGCThreshold = 0.0001
)
func (s *SplitStore) gcHotAfterCompaction() {
// Measure hotstore size, determine if we should do full GC, determine if we can do full GC.
// We should do full GC if
// FullGCFrequency is specified and compaction index matches frequency
// OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target
// We can do full if
// HotstoreMaxSpaceTarget is not specified
// OR total moving space would not exceed 50 GB below target
//
// a) If we should not do full GC => online GC
// b) If we should do full GC and can => moving GC
// c) If we should do full GC and can't => aggressive online GC
getSize := func() int64 {
sizer, ok := s.hot.(bstore.BlockstoreSize)
if ok {
size, err := sizer.Size()
if err != nil {
log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err)
return 0
}
return size
}
log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not")
return 0
}
hotSize := getSize()
copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk
shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold)
shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0
shouldDoFull := shouldTarget || shouldFreq
canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer)
log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk)
log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull)
func (s *SplitStore) gcHotstore() {
var opts []bstore.BlockstoreGCOption
if shouldDoFull && canDoFull {
if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
opts = append(opts, bstore.WithFullGC(true))
} else if shouldDoFull && !canDoFull {
log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget)
log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`")
log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`")
opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold))
}
if err := s.gcBlockstore(s.hot, opts); err != nil {
log.Warnf("error garbage collecting hostore: %s", err)
}
log.Infof("measured hot store size after GC: %d", getSize())
}
func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
if err := s.checkYield(); err != nil {
return err
}
if gc, ok := b.(bstore.BlockstoreGC); ok {
log.Info("garbage collecting blockstore")
startGC := time.Now()
opts = append(opts, bstore.WithCheckFreq(90*time.Second))
opts = append(opts, bstore.WithCheck(s.checkYield))
if err := gc.CollectGarbage(s.ctx, opts...); err != nil {
if err := gc.CollectGarbage(opts...); err != nil {
return err
}
log.Infow("garbage collecting blockstore done", "took", time.Since(startGC))
log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
return nil
}
return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
}
func (s *SplitStore) gcBlockstoreOnce(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
if gc, ok := b.(bstore.BlockstoreGCOnce); ok {
log.Debug("gc blockstore once")
startGC := time.Now()
if err := gc.GCOnce(s.ctx, opts...); err != nil {
return err
}
log.Debugw("gc blockstore once done", "took", time.Since(startGC))
return nil
}
return fmt.Errorf("blockstore doesn't support gc once: %T", b)
}
-591
View File
@@ -1,591 +0,0 @@
package splitstore
import (
"bytes"
"os"
"runtime"
"sync"
"sync/atomic"
"time"
cid "github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
cbg "github.com/whyrusleeping/cbor-gen"
"go.opencensus.io/stats"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/metrics"
)
var (
// PruneOnline is a prune option that instructs PruneChain to use online gc for reclaiming space;
// there is no value associated with this option.
PruneOnlineGC = "splitstore.PruneOnlineGC"
// PruneMoving is a prune option that instructs PruneChain to use moving gc for reclaiming space;
// the value associated with this option is the path of the new coldstore.
PruneMovingGC = "splitstore.PruneMovingGC"
// PruneRetainState is a prune option that instructs PruneChain as to how many finalities worth
// of state to retain in the coldstore.
// The value is an integer:
// - if it is -1 then all state objects reachable from the chain will be retained in the coldstore.
// this is useful for garbage collecting side-chains and other garbage in archival nodes.
// This is the (safe) default.
// - if it is 0 then no state objects that are unreachable within the compaction boundary will
// be retained in the coldstore.
// - if it is a positive integer, then it's the number of finalities past the compaction boundary
// for which chain-reachable state objects are retained.
PruneRetainState = "splitstore.PruneRetainState"
// PruneThreshold is the number of epochs that need to have elapsed
// from the previously pruned epoch to trigger a new prune
PruneThreshold = 7 * build.Finality
)
// GCHotstore runs online GC on the chain state in the hotstore according the to options specified
func (s *SplitStore) GCHotStore(opts api.HotGCOpts) error {
if opts.Moving {
gcOpts := []bstore.BlockstoreGCOption{bstore.WithFullGC(true)}
return s.gcBlockstore(s.hot, gcOpts)
}
gcOpts := []bstore.BlockstoreGCOption{bstore.WithThreshold(opts.Threshold)}
var err error
if opts.Periodic {
err = s.gcBlockstore(s.hot, gcOpts)
} else {
err = s.gcBlockstoreOnce(s.hot, gcOpts)
}
return err
}
// PruneChain instructs the SplitStore to prune chain state in the coldstore, according to the
// options specified.
func (s *SplitStore) PruneChain(opts api.PruneOpts) error {
retainState := opts.RetainState
var gcOpts []bstore.BlockstoreGCOption
if opts.MovingGC {
gcOpts = append(gcOpts, bstore.WithFullGC(true))
}
doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
var retainStateP func(int64) bool
switch {
case retainState > 0:
retainStateP = func(depth int64) bool {
return depth <= int64(CompactionBoundary)+retainState*int64(build.Finality)
}
case retainState < 0:
retainStateP = func(_ int64) bool { return true }
default:
retainStateP = func(depth int64) bool {
return depth <= int64(CompactionBoundary)
}
}
if _, ok := s.cold.(bstore.BlockstoreIterator); !ok {
return xerrors.Errorf("coldstore does not support efficient iteration")
}
return s.pruneChain(retainStateP, doGC)
}
func (s *SplitStore) pruneChain(retainStateP func(int64) bool, doGC func() error) error {
// inhibit compaction while we are setting up
s.headChangeMx.Lock()
defer s.headChangeMx.Unlock()
// take the compaction lock; fail if there is a compaction in progress
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
return xerrors.Errorf("compaction, prune or warmup in progress")
}
// check if we are actually closing first
if atomic.LoadInt32(&s.closing) == 1 {
atomic.StoreInt32(&s.compacting, 0)
return errClosing
}
// ensure that we have compacted at least once
if s.compactionIndex == 0 {
atomic.StoreInt32(&s.compacting, 0)
return xerrors.Errorf("splitstore has not compacted yet")
}
// get the current tipset
curTs := s.chain.GetHeaviestTipSet()
// begin the transaction and go
s.beginTxnProtect()
s.compactType = cold
go func() {
defer atomic.StoreInt32(&s.compacting, 0)
defer s.endTxnProtect()
log.Info("pruning splitstore")
start := time.Now()
s.prune(curTs, retainStateP, doGC)
log.Infow("prune done", "took", time.Since(start))
}()
return nil
}
func (s *SplitStore) prune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) {
log.Debug("waiting for active views to complete")
start := time.Now()
s.viewWait()
log.Debugw("waiting for active views done", "took", time.Since(start))
err := s.doPrune(curTs, retainStateP, doGC)
if err != nil {
log.Errorf("PRUNE ERROR: %s", err)
}
}
func (s *SplitStore) doPrune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) error {
currentEpoch := curTs.Height()
boundaryEpoch := currentEpoch - CompactionBoundary
log.Infow("running prune", "currentEpoch", currentEpoch, "pruneEpoch", s.pruneEpoch)
markSet, err := s.markSetEnv.New("live", s.markSetSize)
if err != nil {
return xerrors.Errorf("error creating mark set: %w", err)
}
defer markSet.Close() //nolint:errcheck
defer s.debug.Flush()
if err := s.checkClosing(); err != nil {
return err
}
// 0. track all protected references at beginning of compaction; anything added later should
// be transactionally protected by the write
log.Info("protecting references with registered protectors")
err = s.applyProtectors()
if err != nil {
return err
}
// 1. mark reachable objects by walking the chain from the current epoch; we keep all messages
// and chain headers; state and reciepts are retained only if it is within retention policy scope
log.Info("marking reachable objects")
startMark := time.Now()
count := new(int64)
err = s.walkChainDeep(curTs, retainStateP,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
mark, err := markSet.Has(c)
if err != nil {
return xerrors.Errorf("error checking markset: %w", err)
}
if mark {
return errStopWalk
}
atomic.AddInt64(count, 1)
return markSet.Mark(c)
})
if err != nil {
return xerrors.Errorf("error marking: %w", err)
}
log.Infow("marking done", "took", time.Since(startMark), "marked", count)
if err := s.checkClosing(); err != nil {
return err
}
// 1.1 protect transactional refs
err = s.protectTxnRefs(markSet)
if err != nil {
return xerrors.Errorf("error protecting transactional refs: %w", err)
}
if err := s.checkClosing(); err != nil {
return err
}
// 2. iterate through the coldstore to collect dead objects
log.Info("collecting dead objects")
startCollect := time.Now()
deadw, err := NewColdSetWriter(s.discardSetPath())
if err != nil {
return xerrors.Errorf("error creating coldset: %w", err)
}
defer deadw.Close() //nolint:errcheck
// some stats for logging
var liveCnt, deadCnt int
err = s.cold.(bstore.BlockstoreIterator).ForEachKey(func(c cid.Cid) error {
// was it marked?
mark, err := markSet.Has(c)
if err != nil {
return xerrors.Errorf("error checking mark set for %s: %w", c, err)
}
if mark {
liveCnt++
return nil
}
// Note: a possible optimization here is to also purge objects that are in the hotstore
// but this needs special care not to purge genesis state, so we don't bother (yet)
// it's dead in the coldstore, mark it as candidate for purge
if err := deadw.Write(c); err != nil {
return xerrors.Errorf("error writing cid to coldstore: %w", err)
}
deadCnt++
return nil
})
if err != nil {
return xerrors.Errorf("error dead objects: %w", err)
}
if err := deadw.Close(); err != nil {
return xerrors.Errorf("error closing deadset: %w", err)
}
stats.Record(s.ctx, metrics.SplitstoreCompactionDead.M(int64(deadCnt)))
log.Infow("dead collection done", "took", time.Since(startCollect))
log.Infow("prune stats", "live", liveCnt, "dead", deadCnt)
if err := s.checkClosing(); err != nil {
return err
}
// now that we have collected dead objects, check for missing references from transactional i/o
// this is carried over from hot compaction for completeness
s.waitForMissingRefs(markSet)
if err := s.checkClosing(); err != nil {
return err
}
deadr, err := NewColdSetReader(s.discardSetPath())
if err != nil {
return xerrors.Errorf("error opening deadset: %w", err)
}
defer deadr.Close() //nolint:errcheck
// 3. Purge dead objects with checkpointing for recovery.
// This is the critical section of prune, whereby any dead object not in the markSet is
// considered already deleted.
// We delete dead objects in batches, holding the transaction lock, where we check the markSet
// again for new references created by the caller.
// After each batch we write a checkpoint to disk; if the process is interrupted before completion
// the process will continue from the checkpoint in the next recovery.
if err := s.beginCriticalSection(markSet); err != nil {
return xerrors.Errorf("error beginning critical section: %w", err)
}
if err := s.checkClosing(); err != nil {
return err
}
checkpoint, err := NewCheckpoint(s.pruneCheckpointPath())
if err != nil {
return xerrors.Errorf("error creating checkpoint: %w", err)
}
defer checkpoint.Close() //nolint:errcheck
log.Info("purging dead objects from the coldstore")
startPurge := time.Now()
err = s.purge(deadr, checkpoint, markSet)
if err != nil {
return xerrors.Errorf("error purging dead objects: %w", err)
}
log.Infow("purging dead objects from coldstore done", "took", time.Since(startPurge))
s.endCriticalSection()
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
}
if err := os.Remove(s.pruneCheckpointPath()); err != nil {
log.Warnf("error removing checkpoint: %s", err)
}
if err := deadr.Close(); err != nil {
log.Warnf("error closing discard set: %s", err)
}
if err := os.Remove(s.discardSetPath()); err != nil {
log.Warnf("error removing discard set: %s", err)
}
// we are done; do some housekeeping
s.endTxnProtect()
err = doGC()
if err != nil {
log.Warnf("error garbage collecting cold store: %s", err)
}
if err := s.setPruneEpoch(boundaryEpoch); err != nil {
return xerrors.Errorf("error saving prune base epoch: %w", err)
}
s.pruneIndex++
err = s.ds.Put(s.ctx, pruneIndexKey, int64ToBytes(s.pruneIndex))
if err != nil {
return xerrors.Errorf("error saving prune index: %w", err)
}
return nil
}
func (s *SplitStore) completePrune() error {
checkpoint, last, err := OpenCheckpoint(s.pruneCheckpointPath())
if err != nil {
return xerrors.Errorf("error opening checkpoint: %w", err)
}
defer checkpoint.Close() //nolint:errcheck
deadr, err := NewColdSetReader(s.discardSetPath())
if err != nil {
return xerrors.Errorf("error opening deadset: %w", err)
}
defer deadr.Close() //nolint:errcheck
markSet, err := s.markSetEnv.Recover("live")
if err != nil {
return xerrors.Errorf("error recovering markset: %w", err)
}
defer markSet.Close() //nolint:errcheck
// PURGE!
s.compactType = cold
log.Info("purging dead objects from the coldstore")
startPurge := time.Now()
err = s.completePurge(deadr, checkpoint, last, markSet)
if err != nil {
return xerrors.Errorf("error purgin dead objects: %w", err)
}
log.Infow("purging dead objects from the coldstore done", "took", time.Since(startPurge))
markSet.EndCriticalSection()
s.compactType = none
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
}
if err := os.Remove(s.pruneCheckpointPath()); err != nil {
log.Warnf("error removing checkpoint: %s", err)
}
if err := deadr.Close(); err != nil {
log.Warnf("error closing deadset: %s", err)
}
if err := os.Remove(s.discardSetPath()); err != nil {
log.Warnf("error removing deadset: %s", err)
}
return nil
}
// like walkChain but peforms a deep walk, using parallel walking with walkObjectLax,
// whereby all extant messages are retained and state roots are retained if they satisfy
// the given predicate.
// missing references are ignored, as we expect to have plenty for snapshot syncs.
func (s *SplitStore) walkChainDeep(ts *types.TipSet, retainStateP func(int64) bool,
f func(cid.Cid) error) error {
visited := cid.NewSet()
toWalk := ts.Cids()
walkCnt := 0
workers := runtime.NumCPU() / 2
if workers < 2 {
workers = 2
}
var wg sync.WaitGroup
workch := make(chan cid.Cid, 16*workers)
errch := make(chan error, workers)
var once sync.Once
defer once.Do(func() { close(workch) })
push := func(c cid.Cid) error {
if !visited.Visit(c) {
return nil
}
select {
case workch <- c:
return nil
case err := <-errch:
return err
}
}
worker := func() {
defer wg.Done()
for c := range workch {
err := s.walkObjectLax(c, f)
if err != nil {
errch <- xerrors.Errorf("error walking object (cid: %s): %w", c, err)
return
}
}
}
for i := 0; i < workers; i++ {
wg.Add(1)
go worker()
}
baseEpoch := ts.Height()
minEpoch := baseEpoch // for progress report
log.Infof("walking at epoch %d", minEpoch)
walkBlock := func(c cid.Cid) error {
if !visited.Visit(c) {
return nil
}
walkCnt++
if err := f(c); err != nil {
return err
}
var hdr types.BlockHeader
err := s.view(c, func(data []byte) error {
return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
})
if err != nil {
return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
}
if hdr.Height < minEpoch {
minEpoch = hdr.Height
if minEpoch%10_000 == 0 {
log.Infof("walking at epoch %d (walked: %d)", minEpoch, walkCnt)
}
}
depth := int64(baseEpoch - hdr.Height)
retainState := retainStateP(depth)
if hdr.Height > 0 {
if err := push(hdr.Messages); err != nil {
return err
}
if retainState {
if err := push(hdr.ParentMessageReceipts); err != nil {
return err
}
}
}
if retainState || hdr.Height == 0 {
if err := push(hdr.ParentStateRoot); err != nil {
return err
}
}
if hdr.Height > 0 {
toWalk = append(toWalk, hdr.Parents...)
}
return nil
}
for len(toWalk) > 0 {
// walking can take a while, so check this with every opportunity
if err := s.checkClosing(); err != nil {
return err
}
select {
case err := <-errch:
return err
default:
}
walking := toWalk
toWalk = nil
for _, c := range walking {
if err := walkBlock(c); err != nil {
return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
}
}
}
once.Do(func() { close(workch) })
wg.Wait()
select {
case err := <-errch:
return err
default:
}
log.Infow("chain walk done", "walked", walkCnt)
return nil
}
// like walkObject but treats missing references laxly; faster version of walkObjectIncomplete
// without an occurs check.
func (s *SplitStore) walkObjectLax(c cid.Cid, f func(cid.Cid) error) error {
if err := f(c); err != nil {
if err == errStopWalk {
return nil
}
return err
}
if c.Prefix().Codec != cid.DagCBOR {
return nil
}
// check this before recursing
if err := s.checkClosing(); err != nil {
return err
}
var links []cid.Cid
err := s.view(c, func(data []byte) error {
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
if ipld.IsNotFound(err) { // not a problem for deep walks
return nil
}
return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
}
for _, c := range links {
err := s.walkObjectLax(c, f)
if err != nil {
return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
}
}
return nil
}
+41 -8
View File
@@ -5,9 +5,10 @@ import (
"runtime"
"sync/atomic"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
var (
@@ -83,7 +84,7 @@ func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
}
func (s *SplitStore) doReify(c cid.Cid) {
var toreify, toforget []cid.Cid
var toreify, totrack, toforget []cid.Cid
defer func() {
s.reifyMx.Lock()
@@ -92,6 +93,9 @@ func (s *SplitStore) doReify(c cid.Cid) {
for _, c := range toreify {
delete(s.reifyInProgress, c)
}
for _, c := range totrack {
delete(s.reifyInProgress, c)
}
for _, c := range toforget {
delete(s.reifyInProgress, c)
}
@@ -101,7 +105,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
defer s.txnLk.RUnlock()
count := 0
_, err := s.walkObjectIncomplete(c, newTmpVisitor(),
err := s.walkObjectIncomplete(c, newTmpVisitor(),
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
@@ -128,10 +132,19 @@ func (s *SplitStore) doReify(c cid.Cid) {
return xerrors.Errorf("error checking hotstore: %w", err)
}
// All reified blocks are tracked at reification start
if has {
toforget = append(toforget, c)
return errStopWalk
if s.txnMarkSet != nil {
hasMark, err := s.txnMarkSet.Has(c)
if err != nil {
log.Warnf("error checking markset: %s", err)
} else if hasMark {
toforget = append(toforget, c)
return errStopWalk
}
} else {
totrack = append(totrack, c)
return errStopWalk
}
}
toreify = append(toreify, c)
@@ -143,7 +156,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
})
if err != nil {
if errors.Is(err, errReifyLimit) {
if xerrors.Is(err, errReifyLimit) {
log.Debug("reification aborted; reify limit reached")
return
}
@@ -178,4 +191,24 @@ func (s *SplitStore) doReify(c cid.Cid) {
}
}
if s.txnMarkSet != nil {
if len(toreify) > 0 {
if err := s.txnMarkSet.MarkMany(toreify); err != nil {
log.Warnf("error marking reified objects: %s", err)
}
}
if len(totrack) > 0 {
if err := s.txnMarkSet.MarkMany(totrack); err != nil {
log.Warnf("error marking tracked objects: %s", err)
}
}
} else {
// if txnActive is false these are noops
if len(toreify) > 0 {
s.trackTxnRefMany(toreify)
}
if len(totrack) > 0 {
s.trackTxnRefMany(totrack)
}
}
}
+14 -19
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package splitstore
import (
@@ -11,20 +11,18 @@ import (
"testing"
"time"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
logging "github.com/ipfs/go-log/v2"
mh "github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
datastore "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
logging "github.com/ipfs/go-log/v2"
mh "github.com/multiformats/go-multihash"
)
func init() {
@@ -38,7 +36,6 @@ func init() {
func testSplitStore(t *testing.T, cfg *Config) {
ctx := context.Background()
chain := &mockChain{t: t}
fmt.Printf("Config: %v\n", cfg)
// the myriads of stores
ds := dssync.MutexWrap(datastore.NewMapDatastore())
@@ -226,7 +223,7 @@ func TestSplitStoreCompaction(t *testing.T) {
//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
testSplitStore(t, &Config{MarkSetType: "map", UniversalColdBlocks: true})
testSplitStore(t, &Config{MarkSetType: "map"})
}
func TestSplitStoreCompactionWithBadger(t *testing.T) {
@@ -238,7 +235,7 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
t.Cleanup(func() {
badgerMarkSetBatchSize = bs
})
testSplitStore(t, &Config{MarkSetType: "badger", UniversalColdBlocks: true})
testSplitStore(t, &Config{MarkSetType: "badger"})
}
func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
@@ -284,7 +281,7 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
path := t.TempDir()
// open the splitstore
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
@@ -423,7 +420,7 @@ func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.
path := t.TempDir()
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
@@ -523,7 +520,7 @@ func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blocks
path := t.TempDir()
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
@@ -699,7 +696,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
blk, ok := b.set[b.keyOf(cid)]
if !ok {
return nil, ipld.ErrNotFound{Cid: cid}
return nil, blockstore.ErrNotFound
}
return blk, nil
}
@@ -757,8 +754,6 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
return nil
}
func (b *mockStore) Flush(context.Context) error { return nil }
func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
return nil, errors.New("not implemented")
}
+3 -2
View File
@@ -3,10 +3,11 @@ package splitstore
import (
"encoding/binary"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"github.com/filecoin-project/go-state-types/abi"
)
+8 -8
View File
@@ -5,13 +5,13 @@ import (
"sync/atomic"
"time"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
)
@@ -21,14 +21,14 @@ var (
WarmupBoundary = build.Finality
)
// warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore;
// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
// this is necessary when we sync from a snapshot or when we enable the splitstore
// on top of an existing blockstore (which becomes the coldstore).
func (s *SplitStore) warmup(curTs *types.TipSet) error {
if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
return xerrors.Errorf("error locking compaction")
}
s.compactType = warmup
go func() {
defer atomic.StoreInt32(&s.compacting, 0)
@@ -88,7 +88,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
blk, err := s.cold.Get(s.ctx, c)
if err != nil {
if ipld.IsNotFound(err) {
if err == bstore.ErrNotFound {
atomic.AddInt64(missing, 1)
return errStopWalk
}
@@ -110,7 +110,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
mx.Unlock()
return nil
}, func(cid.Cid) error { return nil })
})
if err != nil {
return err
+1 -1
View File
@@ -3,7 +3,7 @@ package splitstore
import (
"sync"
"github.com/ipfs/go-cid"
cid "github.com/ipfs/go-cid"
)
// ObjectVisitor is an interface for deduplicating objects during walks
+1 -3
View File
@@ -4,8 +4,8 @@ import (
"context"
"sync"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
)
// NewMemorySync returns a thread-safe in-memory blockstore.
@@ -20,8 +20,6 @@ type SyncBlockstore struct {
bs MemBlockstore // specifically use a memStore to save indirection overhead.
}
func (*SyncBlockstore) Flush(context.Context) error { return nil }
func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
m.mu.Lock()
defer m.mu.Unlock()
+8 -20
View File
@@ -6,9 +6,8 @@ import (
"sync"
"time"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/raulk/clock"
"go.uber.org/multierr"
)
@@ -93,16 +92,6 @@ func (t *TimedCacheBlockstore) rotate() {
t.mu.Unlock()
}
func (t *TimedCacheBlockstore) Flush(ctx context.Context) error {
t.mu.Lock()
defer t.mu.Unlock()
if err := t.active.Flush(ctx); err != nil {
return err
}
return t.inactive.Flush(ctx)
}
func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
// Don't check the inactive set here. We want to keep this block for at
// least one interval.
@@ -123,7 +112,7 @@ func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback fun
// calling an arbitrary callback while holding a lock.
t.mu.RLock()
block, err := t.active.Get(ctx, k)
if ipld.IsNotFound(err) {
if err == ErrNotFound {
block, err = t.inactive.Get(ctx, k)
}
t.mu.RUnlock()
@@ -138,7 +127,7 @@ func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block
t.mu.RLock()
defer t.mu.RUnlock()
b, err := t.active.Get(ctx, k)
if ipld.IsNotFound(err) {
if err == ErrNotFound {
b, err = t.inactive.Get(ctx, k)
}
return b, err
@@ -148,7 +137,7 @@ func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, err
t.mu.RLock()
defer t.mu.RUnlock()
size, err := t.active.GetSize(ctx, k)
if ipld.IsNotFound(err) {
if err == ErrNotFound {
size, err = t.inactive.GetSize(ctx, k)
}
return size, err
@@ -186,12 +175,11 @@ func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, e
defer t.mu.RUnlock()
ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
for _, b := range t.active {
ch <- b.Cid()
for c := range t.active {
ch <- c
}
for _, b := range t.inactive {
c := b.Cid()
if _, ok := t.active[string(c.Hash())]; ok {
for c := range t.inactive {
if _, ok := t.active[c]; ok {
continue
}
ch <- c
+4 -3
View File
@@ -1,4 +1,4 @@
// stm: #unit
//stm: #unit
package blockstore
import (
@@ -6,10 +6,11 @@ import (
"testing"
"time"
"github.com/ipfs/go-cid"
blocks "github.com/ipfs/go-libipfs/blocks"
"github.com/raulk/clock"
"github.com/stretchr/testify/require"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
)
func TestTimedCacheBlockstoreSimple(t *testing.T) {
+8 -17
View File
@@ -3,18 +3,18 @@ package blockstore
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
)
type unionBlockstore []Blockstore
// Union returns an unioned blockstore.
//
// - Reads return from the first blockstore that has the value, querying in the
// supplied order.
// - Writes (puts and deletes) are broadcast to all stores.
// * Reads return from the first blockstore that has the value, querying in the
// supplied order.
// * Writes (puts and deltes) are broadcast to all stores.
//
func Union(stores ...Blockstore) Blockstore {
return unionBlockstore(stores)
}
@@ -30,7 +30,7 @@ func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err er
func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
for _, bs := range m {
if blk, err = bs.Get(ctx, cid); err == nil || !ipld.IsNotFound(err) {
if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
break
}
}
@@ -39,7 +39,7 @@ func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block
func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
for _, bs := range m {
if err = bs.View(ctx, cid, callback); err == nil || !ipld.IsNotFound(err) {
if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
break
}
}
@@ -48,22 +48,13 @@ func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]
func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
for _, bs := range m {
if size, err = bs.GetSize(ctx, cid); err == nil || !ipld.IsNotFound(err) {
if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
break
}
}
return size, err
}
func (m unionBlockstore) Flush(ctx context.Context) (err error) {
for _, bs := range m {
if err = bs.Flush(ctx); err != nil {
break
}
}
return err
}
func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
for _, bs := range m {
if err = bs.Put(ctx, block); err != nil {
+2 -3
View File
@@ -1,11 +1,11 @@
// stm: #unit
//stm: #unit
package blockstore
import (
"context"
"testing"
blocks "github.com/ipfs/go-libipfs/blocks"
blocks "github.com/ipfs/go-block-format"
"github.com/stretchr/testify/require"
)
@@ -13,7 +13,6 @@ var (
b0 = blocks.NewBlock([]byte("abc"))
b1 = blocks.NewBlock([]byte("foo"))
b2 = blocks.NewBlock([]byte("bar"))
b3 = blocks.NewBlock([]byte("baz"))
)
func TestUnionBlockstore_Get(t *testing.T) {
-26
View File
@@ -1,26 +0,0 @@
# Bundles
This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed
tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the
lotus binary.
## Updating
To update, run the `./pack.sh` script. For example, the following will pack the [builtin actors release](https://github.com/filecoin-project/builtin-actors/releases) `dev/20220602` into the `v8` tarfile.
```bash
./pack.sh v8 dev/20220602
```
This will:
1. Download the actors bundles and pack them into the appropriate tarfile (`$VERSION.tar.zst`).
2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file for _all_ network versions (all `*.tar.zst` files in this directory).
## Overriding
To build a bundle, but specify a different release/tag for a specific network, append `$network=$alternative_release` on the command line. For example:
```bash
./pack.sh v8 dev/20220602 mainnet=v8.0.0 calibrationnet=v8.0.0-rc.1
```
-55
View File
@@ -1,55 +0,0 @@
#!/bin/bash
NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs calibrationnet)
set -e
if [[ $# -lt 2 ]]; then
echo "Usage: $0 VERSION RELEASE [NETWORK=RELEASE_OVERRIDE]..." >&2
echo "expected at least two arguments, an actors version (e.g., v8), an actors release, and any number of release overrides." >&2
exit 1
fi
VERSION="$1" # actors version
RELEASE="$2" # actors release name
RELEASE_OVERRIDES=("${@:3}")
echo "Downloading bundles for actors version ${VERSION} release ${RELEASE}"
echo "With release overrides ${RELEASE_OVERRIDES[*]}"
TARGET_FILE="$(pwd)/${VERSION}.tar.zst"
WORKDIR=$(mktemp -d -t "actor-bundles-${VERSION}.XXXXXXXXXX")
trap 'rm -rf -- "$WORKDIR"' EXIT
encode_release() {
jq -rn --arg release "$1" '$release | @uri'
}
pushd "${WORKDIR}"
for network in "${NETWORKS[@]}"; do
release="$RELEASE"
# Ideally, we'd use an associative array (map). But that's not supported on macos.
for override in "${RELEASE_OVERRIDES[@]}"; do
if [[ "${network}" = "${override%%=*}" ]]; then
release="${override#*=}"
break
fi
done
encoded_release="$(encode_release "$release")"
echo "Downloading $release for network $network."
wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256}
done
echo "Checking the checksums..."
sha256sum -c -- *.sha256
echo "Packing..."
rm -f -- "$TARGET_FILE"
tar -cf "$TARGET_FILE" --use-compress-program "zstd -19" -- *.car
popd
echo "Generating metadata..."
make -C ../../ VERSION="$VERSION" RELEASE="$RELEASE" RELEASE_OVERRIDES="${RELEASE_OVERRIDES[*]}" bundle-gen
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+2 -2
View File
@@ -6,9 +6,9 @@ import (
"path"
"strings"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/filecoin-project/lotus/lib/addrutil"
"github.com/libp2p/go-libp2p-core/peer"
)
//go:embed bootstrap

Some files were not shown because too many files have changed in this diff Show More