update ffi to include the multi group sdr change

.circleci/gen.go

@@ -10,25 +10,11 @@ import (
 	"text/template"
 )
 
-var GoVersion = "" // from init below. Ex: 1.19.7
-
 //go:generate go run ./gen.go ..
 
 //go:embed template.yml
 var templateFile embed.FS
 
-func init() {
-	b, err := os.ReadFile("../go.mod")
-	if err != nil {
-		panic("cannot find go.mod in parent folder")
-	}
-	for _, line := range strings.Split(string(b), "\n") {
-		if strings.HasPrefix(line, "go ") {
-			GoVersion = line[3:]
-		}
-	}
-}
-
 type (
 	dirs  = []string
 	suite = string
@@ -79,8 +65,6 @@ func main() {
 			if err != nil {
 				panic(err)
 			}
-			// Redundantly flag both absolute and relative paths as excluded
-			excluded[filepath.Join(repo, s)] = struct{}{}
 			excluded[e] = struct{}{}
 		}
 	}
@@ -122,13 +106,10 @@ func main() {
 
 	// form the input data.
 	type data struct {
-		Networks   []string
 		ItestFiles []string
 		UnitSuites map[string]string
-		GoVersion  string
 	}
 	in := data{
-		Networks:   []string{"mainnet", "butterflynet", "calibnet", "debug"},
 		ItestFiles: itests,
 		UnitSuites: func() map[string]string {
 			ret := make(map[string]string)
@@ -140,7 +121,6 @@ func main() {
 			}
 			return ret
 		}(),
-		GoVersion: GoVersion,
 	}
 
 	out, err := os.Create("./config.yml")

.codecov.yml

@@ -0,0 +1,73 @@
+ignore:
+  # Auto generated
+  - "^.*_gen.go$"
+  - "^.*/mock_full.go$"
+  # Old actors.
+  - "^chain/actors/builtin/[^/]*/(message|state|v)[0-4]\\.go$" # We test the latest version only.
+  # Tests
+  - "api/test/**"
+  - "conformance/**"
+  # Generators
+  - "gen/**"
+  - "chain/actors/agen/**"
+  # Non-critical utilities
+  - "api/docgen/**"
+  - "api/docgen-openrpc/**"
+coverage:
+  status:
+    patch: off
+    project:
+      tools-and-tests:
+        target: auto
+        threshold: 1%
+        informational: true
+        paths:
+          - "testplans"
+          - "tools"
+          - "system"
+          - "snap"
+          - "lotuspond"
+          - "conformance"
+          - "scripts"
+          - "gen"
+          - "build"
+      markets: 
+        target: auto
+        threshold: 1%
+        informational: false        
+        paths: 
+          - "markets"
+          - "paychmgr"
+      miner: 
+        target: auto
+        threshold: 1.5%
+        informational: false       
+        paths: 
+          - "miner"
+          - "storage"
+      chain:
+        target: auto
+        threshold: 1%
+        informational: false
+        paths:
+          - "chain"
+      node:
+        target: auto
+        threshold: 1%
+        informational: false
+        paths: 
+          - "node"
+          - "blockstore"
+          - "metrics"
+          - "lib"
+          - "genesis"
+          - "gateway"
+          - "api"
+          - "journal"
+      cli: 
+        target: auto
+        threshold: 1%
+        informational: true
+        paths: 
+          - "cli"
+          - "cmd"

.gitattributes

@@ -1,2 +0,0 @@
-chain/actors/builtin/*/v* linguist-generated=true
-chain/actors/builtin/*/message* linguist-generated=true

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -9,9 +9,15 @@ body:
     options:
       - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
         required: true
+      - label: This is **not** a question or a support request. If you have any lotus related questions, please ask in the [lotus forum](https://github.com/filecoin-project/lotus/discussions).
+        required: true
+      - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
+        required: true
+      - label: This is **not** an enhancement request. If it is, please file a [improvement suggestion](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Fenhancement&template=enhancement.yml) instead.
+        required: true
       - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
         required: true
-      - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
+      - label: I am running the [`Latest release`](https://github.com/filecoin-project/lotus/releases), or the most recent RC(release canadiate) for the upcoming release or the dev branch(master), or have an issue updating to any of these.
         required: true
       - label: I did not make any code changes to lotus.
         required: false
@@ -22,11 +28,19 @@ body:
     options:
       - label: lotus daemon - chain sync
         required: false
-      - label: lotus fvm/fevm - Lotus FVM and FEVM interactions
+      - label: lotus miner - mining and block production
         required: false
       - label: lotus miner/worker - sealing
         required: false
-      - label: lotus miner - proving(WindowPoSt/WinningPoSt)
+      - label: lotus miner - proving(WindowPoSt)
+        required: false
+      - label: lotus miner/market - storage deal
+        required: false
+      - label: lotus miner/market - retrieval deal
+        required: false
+      - label: lotus miner/market - data transfer
+        required: false
+      - label: lotus client
         required: false
       - label: lotus JSON-RPC API
         required: false
@@ -42,33 +56,22 @@ body:
     description: Enter the output of `lotus version` and `lotus-miner version` if applicable.
     placeholder: |
       e.g. 
-      Daemon:  1.19.0+mainnet+git.64059ca87+api1.5.0
+      Daemon:1.11.0-rc2+debug+git.0519cd371.dirty+api1.3.0 
-      Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
+      Local: lotus version 1.11.0-rc2+debug+git.0519cd371.dirty
   validations:
     required: true
-- type: textarea
-  id: ReproSteps
-  attributes:
-    label: Repro Steps
-    description: "Steps to reproduce the behavior"
-    value: |
-      1. Run '...'
-      2. Do '...'
-      3. See error '...'
-      ...
-  validations:
-    required: false
 - type: textarea
   id: Description
   attributes:
     label: Describe the Bug
     description: |
       This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
-      * What you were doing when you experienced the bug?
+      * What you were doding when you experienced the bug?
       * Any *error* messages you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
       * What is the expected behaviour?
       * For sealing issues, include the output of `lotus-miner sectors status --log <sectorId>` for the failed sector(s).
       * For proving issues, include the output of `lotus-miner proving` info.
+      * For deal making issues, include the output of `lotus client list-deals -v` and/or `lotus-miner storage-deals|retrieval-deals|data-transfers list [-v]` commands for the deal(s) in question.
   validations:
     required: true
 - type: textarea
@@ -78,8 +81,20 @@ body:
     render: text
     description: |
       Please provide debug logs of the problem, remember you can get set log level control for:
-      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://lotus.filecoin.io/lotus/configure/defaults/#log-level-control).
+      * lotus: use `lotus log list` to get all log systems available and set level by `lotus log set-level`. An example can be found [here](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/#log-level-control).
       * lotus-miner:`lotus-miner log list` to get all log systems available and set level by `lotus-miner log set-level
-      If you don't provide detailed logs when you raise the issue it will almost certainly be the first request we make before furthur diagnosing the problem.
+      If you don't provide detailed logs when you raise the issue it will almost certainly be the first request I make before furthur diagnosing the problem.
   validations:
     required: true
+- type: textarea
+  id: RepoSteps
+  attributes:
+    label: Repo Steps
+    description: "Steps to reproduce the behavior"
+    value: |
+      1. Run '...'
+      2. Do '...'
+      3. See error '...'
+      ...
+  validations:
+    required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Ask a question about Lotus or get support
-    url: https://github.com/filecoin-project/lotus/discussions/new/choose
-    about: Ask a question or request support for using Lotus
-  - name: Filecoin protocol feature or enhancement
-    url: https://github.com/filecoin-project/FIPs/discussions/new/choose
-    about: Write a discussion in the Filecoin Improvement Proposal repo

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -7,7 +7,13 @@ body:
     label: Checklist
     description: Please check off the following boxes before continuing to create an improvement suggestion!
     options:
-      - label: I **have** a specific, actionable, and well motivated improvement to an existing lotus feature.
+      - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is,  please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
+        required: true
+      - label: This is **not** a new feature request. If it is, please file a [feature request](https://github.com/filecoin-project/lotus/issues/new?assignees=&labels=need%2Ftriage%2Ckind%2Ffeature&template=feature_request.yml) instead.
+        required: true
+      - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
+        required: true
+      - label: I **have** a specific, actionable, and well motivated improvement to propose.
         required: true
 - type: checkboxes
   attributes:
@@ -16,11 +22,19 @@ body:
     options:
       - label: lotus daemon - chain sync
         required: false
-      - label: lotus fvm/fevm - Lotus FVM and FEVM interactions
+      - label: lotus miner - mining and block production
         required: false
       - label: lotus miner/worker - sealing
         required: false
-      - label: lotus miner - proving(WindowPoSt/WinningPoSt)
+      - label: lotus miner - proving(WindowPoSt)
+        required: false
+      - label: lotus miner/market - storage deal
+        required: false
+      - label: lotus miner/market - retrieval deal
+        required: false
+      - label: lotus miner/market - data transfer
+        required: false
+      - label: lotus client
         required: false
       - label: lotus JSON-RPC API
         required: false
@@ -31,17 +45,9 @@ body:
 - type: textarea
   id: request
   attributes:
-    label: Enhancement Suggestion
+    label: Improvement Suggestion
-    description: A clear and concise description of the suggested enhancement?
+    description: A clear and concise description of what the motivation or the current problem is and what is the suggested improvement?
-    placeholder: Ex. Currently lotus... However it would be great if [enhancement] was implemented... With the ability to...
+    placeholder: Ex. Currently lotus... However, as a storage provider, I'd like...
-  validations:
-    required: true
-- type: textarea
-  id: request
-  attributes:
-    label: Use-Case
-    description: How would this enhancement help you?
-    placeholder: Ex. With the [enhancement] node operators would be able to... For Storage Providers it would enable...
   validations:
     required: true
 

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,6 +7,8 @@ body:
     label: Checklist
     description: Please check off the following boxes before continuing to create a new feature request!
     options:
+      - label: This is **not** a new feature or an enhancement to the Filecoin protocol. If it is,  please open an [FIP issue](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0001.md).
+        required: true
       - label: This is **not** brainstorming ideas. If you have an idea you'd like to discuss, please open a new discussion on [the lotus forum](https://github.com/filecoin-project/lotus/discussions/categories/ideas) and select the category as `Ideas`.
         required: true
       - label: I **have** a specific, actionable, and well motivated feature request to propose.
@@ -18,11 +20,19 @@ body:
     options:
       - label: lotus daemon - chain sync
         required: false
-      - label: lotus fvm/fevm - Lotus FVM and FEVM interactions
+      - label: lotus miner - mining and block production
         required: false
       - label: lotus miner/worker - sealing
         required: false
-      - label: lotus miner - proving(WindowPoSt/WinningPoSt)
+      - label: lotus miner - proving(WindowPoSt)
+        required: false
+      - label: lotus miner/market - storage deal
+        required: false
+      - label: lotus miner/market - retrieval deal
+        required: false
+      - label: lotus miner/market - data transfer
+        required: false
+      - label: lotus client
         required: false
       - label: lotus JSON-RPC API
         required: false
@@ -46,7 +56,7 @@ body:
   validations:
     required: true
 - type: textarea
-  id: alternatives
+  id: alternates
   attributes:
     label: Describe alternatives you've considered
     description: A clear and concise description of any alternative solutions or features you've considered.
@@ -59,3 +69,4 @@ body:
     description: Add any other context, design docs or screenshots about the feature request here.
   validations:
     required: false
+

.github/ISSUE_TEMPLATE/service_developer_bug_report.yml

@@ -1,83 +0,0 @@
-name: "Bug Report - developer/service provider"
-description: "Bug report template about FEVM/FVM for developers/service providers"
-labels: [need/triage, kind/bug, area/fevm]
-body:
-- type: checkboxes
-  attributes:
-    label: Checklist
-    description: Please check off the following boxes before continuing to file a bug report!
-    options:
-      - label: This is **not** a security-related bug/issue. If it is, please follow please follow the [security policy](https://github.com/filecoin-project/lotus/security/policy).
-        required: true
-      - label: I **have** searched on the [issue tracker](https://github.com/filecoin-project/lotus/issues) and the [lotus forum](https://github.com/filecoin-project/lotus/discussions), and there is no existing related issue or discussion.
-        required: true
-      - label: I did not make any code changes to lotus.
-        required: false
-- type: checkboxes
-  attributes:
-    label: Lotus component
-    description: Please select the lotus component you are filing a bug for
-    options:
-      - label: lotus Ethereum RPC
-        required: false
-      - label: lotus FVM - Lotus FVM interactions
-        required: false
-      - label: FEVM tooling
-        required: false
-      - label:  Other
-        required: false
-- type: textarea
-  id: version
-  attributes:
-    label: Lotus Version
-    render: text
-    description: Enter the output of `lotus version` if applicable.
-    placeholder: |
-      e.g. 
-      Daemon:  1.19.0+mainnet+git.64059ca87+api1.5.0
-      Local: lotus-miner version 1.19.0+mainnet+git.64059ca87
-  validations:
-    required: true
-- type: textarea
-  id: repro
-  attributes:
-    label: Repro Steps
-    description: "Steps to reproduce the behavior"
-    value: |
-      1. Run '...'
-      2. Do '...'
-      3. See error '...'
-      ...
-  validations:
-    required: false
-- type: textarea
-  id: Description
-  attributes:
-    label: Describe the Bug
-    description: |
-      This is where you get to tell us what went wrong, when doing so, please try to provide a clear and concise description of the bug with all related information:
-      * What you were doing when you experienced the bug? What are you trying to build?
-      * Any *error* messages and logs you saw, *where* you saw them, and what you believe may have caused them (if you have any ideas).
-      * What is the expected behaviour? Links to the actual code?
-  validations:
-    required: true
-- type: textarea
-  id: toolingInfo
-  attributes:
-    label: Tooling
-    render: text
-    description: |
-      What kind of tooling are you using:
-      * Are you using ether.js, Alchemy, Hardhat, etc.
-  validations:
-    required: true
-- type: textarea
-  id: extraInfo
-  attributes:
-    label: Configuration Options
-    render: text
-    description: |
-      Please provide your updated FEVM related configuration options, or custome enviroment variables related to Lotus FEVM
-      * lotus: use `lotus config updated` to get your configuration options, and copy the [FEVM] section
-  validations:
-    required: true

.github/ISSUE_TEMPLATE/task.md

@@ -1,31 +0,0 @@
----
-name: New Task
-about: A larger yet well-scoped task
-title: '<title>'
-labels: Needs Triage
-assignees: ''
-
----
-
-## User Story
-<!-- Why? -->
-
-## Acceptance Criteria
-<!-- What? -->
-<!-- add description-->
-
-```[tasklist]
-### Deliverables
-
-```
-
-## Technical Breakdown
-```[tasklist]
-### Development
-
-```
-
-```[tasklist]
-### Testing
-
-```

.github/pull_request_template.md

@@ -1,24 +1,21 @@
 ## Related Issues
-<!-- Link issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made -->
+<!-- link all issues that this PR might resolve/fix. If an issue doesn't exist, include a brief motivation for the change being made.-->
 
 ## Proposed Changes
-<!-- A clear list of the changes being made -->
+<!-- provide a clear list of the changes being made-->
+
 
 ## Additional Info
-<!-- Callouts, links to documentation, and etc -->
+<!-- callouts, links to documentation, and etc-->
 
 ## Checklist
 
 Before you mark the PR ready for review, please make sure that:
-
+- [ ] All commits have a clear commit message.
-- [ ] Commits have a clear commit message.
+- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
-- [ ] PR title is in the form of of `<PR type>: <area>: <change being made>`
+    - example: ` fix: mempool: Introduce a cache for valid signatures`
-  - example: ` fix: mempool: Introduce a cache for valid signatures`
+    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
-  - `PR type`: fix, feat, build, chore, ci, docs, perf, refactor, revert, style, test
+    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
-  - `area`, e.g. api, chain, state, market, mempool, multisig, networking, paych, proving, sealing, wallet, deps
+- [ ] This PR has tests for new functionality or change in behaviour
-- [ ] If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section.
+- [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
-- [ ] New features have usage guidelines and / or documentation updates in
-  - [ ] [Lotus Documentation](https://lotus.filecoin.io)
-  - [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
-- [ ] Tests exist for new functionality or change in behavior
 - [ ] CI is green

.github/workflows/codeql-analysis.yml

@@ -13,14 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches:
+    branches: [ master ]
-      - master
-      - 'release/*'
   pull_request:
     # The branches below must be a subset of the branches above
-    branches:
+    branches: [ master ]
-      - master
-      - 'release/*'
 
 jobs:
   analyze:
@@ -37,17 +33,17 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v1
       with:
-        go-version: '1.18.8'
+        go-version: '1.17.9'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v1
       with:
-        languages: go
+        languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
@@ -56,7 +52,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -70,4 +66,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v1

.github/workflows/testground-on-push.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: testground run
-        uses: testground/testground-github-action@v1
+        uses: coryschwartz/testground-github-action@v1.1
         with:
           backend_addr: ${{ matrix.backend_addr }}
           backend_proto: ${{ matrix.backend_proto }}

.gitignore

@@ -1,3 +1,6 @@
+/AppDir
+/appimage-builder-cache
+*.AppImage
 /lotus
 /lotus-miner
 /lotus-worker
@@ -6,7 +9,7 @@
 /lotus-chainwatch
 /lotus-shed
 /lotus-sim
-/lotus-provider
+/lotus-pond
 /lotus-townhall
 /lotus-fountain
 /lotus-stats
@@ -18,6 +21,8 @@
 /docgen-md
 /docgen-openrpc
 /bench.json
+/lotuspond/front/node_modules
+/lotuspond/front/build
 /cmd/lotus-townhall/townhall/node_modules
 /cmd/lotus-townhall/townhall/build
 /cmd/lotus-townhall/townhall/package-lock.json
@@ -36,22 +41,11 @@ build/paramfetch.sh
 /darwin
 /linux
 *.snap
+build/builtin-actors
 
 *-fuzz.zip
 /chain/types/work_msg/
 bin/ipget
 bin/tmp/*
 .idea
-.vscode
 scratchpad
-
-build/builtin-actors/v*
-build/builtin-actors/*.car
-
-dist/
-
-
-# The following files are checked into git and result
-# in dirty git state if removed from the docker context
-!extern/filecoin-ffi/rust/filecoin.pc
-!extern/test-vectors

.golangci.yml

@@ -12,9 +12,9 @@ linters:
     - unconvert
     - staticcheck
     - varcheck
+    - structcheck
     - deadcode
     - scopelint
-    - unused
 
 # We don't want to skip builtin/
 skip-dirs-use-default: false
@@ -43,6 +43,9 @@ issues:
 
   exclude-use-default: false
   exclude-rules:
+    - path: lotuspond
+      linters:
+        - errcheck
 
     - path: node/modules/lp2p
       linters:

.goreleaser.yaml

@@ -1,109 +0,0 @@
-project_name: lotus
-
-universal_binaries:
-  - id: lotus
-    replace: true
-    name_template: lotus
-  - id: lotus-miner
-    replace: true
-    name_template: lotus-miner
-  - id: lotus-worker
-    replace: true
-    name_template: lotus-worker
-
-builds:
-  - id: lotus
-    binary: lotus
-    builder: prebuilt
-    goos:
-      - darwin
-      - linux
-    goarch:
-      - amd64
-      - arm64
-    goamd64:
-      - v1
-    ignore:
-      - goos: linux
-        goarch: arm64
-    prebuilt:
-      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus
-  - id: lotus-miner
-    binary: lotus-miner
-    builder: prebuilt
-    goos:
-      - darwin
-      - linux
-    goarch:
-      - amd64
-      - arm64
-    goamd64:
-      - v1
-    ignore:
-      - goos: linux
-        goarch: arm64
-    prebuilt:
-      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner
-  - id: lotus-worker
-    binary: lotus-worker
-    builder: prebuilt
-    goos:
-      - darwin
-      - linux
-    goarch:
-      - amd64
-      - arm64
-    goamd64:
-      - v1
-    ignore:
-      - goos: linux
-        goarch: arm64
-    prebuilt:
-      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker
-
-archives:
-  - id: primary
-    format: tar.gz
-    wrap_in_directory: true
-    name_template: "{{ .ProjectName }}_v{{ .Version }}_{{ .Os }}_{{ .Arch }}"
-    files:
-      # this is a dumb but required hack so it doesn't include the default files
-      # https://github.com/goreleaser/goreleaser/issues/602
-      - _n_o_n_e_*
-
-release:
-  github:
-    owner: filecoin-project
-    name: lotus
-  prerelease: auto
-  name_template: "v{{.Version}}"
-
-brews:
-  - tap:
-      owner: filecoin-project
-      name: homebrew-lotus
-      branch: master
-    ids:
-      - primary
-    install: |
-      bin.install "lotus"
-      bin.install "lotus-miner"
-      bin.install "lotus-worker"
-    test: |
-      system "#{bin}/lotus --version"
-      system "#{bin}/lotus-miner --version"
-      system "#{bin}/lotus-worker --version"
-    folder: Formula
-    homepage: "https://filecoin.io"
-    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
-    license: MIT
-    skip_upload: auto
-    dependencies:
-      - name: hwloc
-
-# produced manually so we can include cid checksums
-checksum:
-  disable: true
-
-snapshot:
-  name_template: "{{ .Version }}"

AppDir/usr/share/icons/icon.svg

@@ -0,0 +1 @@
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0" y="0" viewBox="0 0 127 127" xml:space="preserve" enable-background="new 0 0 127 127"><style type="text/css">.st0{fill:#00d2d6}.st1{fill:#fff}</style><g><path class="st0" d="M63.5,127C28.5,127.1-0.2,98.4,0,63.2C0.2,28.3,28.6-0.2,63.9,0c34.8,0.2,63.3,28.7,63.1,64 C126.7,98.7,98.5,127.1,63.5,127z M71.4,57.6c5.5,0.8,11,1.5,16.5,2.3c0.5-1.7,0.9-3.1,1.3-4.7c-5.7-0.8-11.2-1.7-17.1-2.5 c2-7,3.7-13.7,5.8-20.2c0.7-2.2,2.3-4.2,3.9-5.9c2.1-2.2,5-1.7,6.8,0.7c0.7,1,1.4,2.1,2.3,2.9c1.1,1.1,2.8,1.6,4,0.6 c0.8-0.7,0.7-2.4,0.8-3.6c0-0.5-0.6-1.1-1-1.6c-2-2.3-4.7-3.1-7.5-3.2c-6.3-0.3-10.9,3-14.5,7.8c-3.5,4.8-5.1,10.5-6.8,16.2 c-0.5,1.6-0.9,3.3-1.4,5.1c-6.2-0.9-12.1-1.7-18.2-2.6c-0.2,1.6-0.4,3.2-0.6,4.8c6,0.9,11.8,1.8,17.8,2.7c-0.8,3.4-1.5,6.5-2.3,9.7 c-5.8-0.8-11.4-1.6-17-2.4c-0.2,1.8-0.4,3.2-0.6,4.8c5.6,0.9,11,1.7,16.5,2.5c0,0.6,0.1,1,0,1.3c-1.7,7.4-3.4,14.8-5.3,22.2 c-0.9,3.5-2.4,6.9-5.3,9.3c-2.4,2-5,1.7-6.8-0.8c-0.8-1.1-1.5-2.5-2.6-3.3c-0.8-0.6-2.5-0.9-3.1-0.5c-0.9,0.7-1.5,2.2-1.4,3.3 c0.1,1,1,2.2,1.9,2.8c3,2.3,6.5,2.6,10,1.9c5.9-1.2,10.1-4.9,12.7-10.1c2-4.1,3.6-8.5,5-12.9c1.3-4,2.2-8,3.3-12.2 c5.8,0.8,11.5,1.7,17.3,2.5c0.5-1.7,0.9-3.2,1.4-4.8c-6.1-0.9-11.9-1.7-17.7-2.6C70.1,64,70.7,60.9,71.4,57.6z"/><path class="st1" d="M71.4,57.6c-0.7,3.3-1.3,6.4-2,9.8c5.9,0.9,11.7,1.7,17.7,2.6c-0.5,1.6-0.9,3.1-1.4,4.8 c-5.8-0.8-11.5-1.7-17.3-2.5c-1.1,4.2-2,8.3-3.3,12.2c-1.4,4.4-3,8.7-5,12.9c-2.6,5.2-6.8,8.9-12.7,10.1c-3.5,0.7-7,0.4-10-1.9 c-0.9-0.7-1.8-1.8-1.9-2.8c-0.1-1.1,0.5-2.7,1.4-3.3c0.6-0.5,2.3-0.1,3.1,0.5c1.1,0.8,1.8,2.1,2.6,3.3c1.8,2.5,4.4,2.9,6.8,0.8 c2.9-2.5,4.4-5.8,5.3-9.3c1.9-7.3,3.6-14.8,5.3-22.2c0.1-0.3,0-0.7,0-1.3c-5.4-0.8-10.8-1.7-16.5-2.5c0.2-1.6,0.4-3,0.6-4.8 c5.6,0.8,11.1,1.6,17,2.4c0.8-3.2,1.5-6.4,2.3-9.7c-6-0.9-11.7-1.8-17.8-2.7c0.2-1.6,0.4-3.2,0.6-4.8c6.1,0.9,12,1.7,18.2,2.6 c0.5-1.8,0.9-3.5,1.4-5.1c1.7-5.6,3.2-11.3,6.8-16.2c3.6-4.9,8.1-8.1,14.5-7.8c2.8,0.1,5.5,0.9,7.5,3.2c0.4,0.5,1,1.1,1,1.6 c-0.1,1.2,0,2.9-0.8,3.6c-1.1,1.1-2.8,0.5-4-0.6c-0.9-0.9-1.6-1.9-2.3-2.9c-1.8-2.4-4.7-2.9-6.8-0.7c-1.6,1.7-3.2,3.7-3.9,5.9 C75.7,39.4,74,46,72,53c5.9,0.9,11.4,1.7,17.1,2.5c-0.5,1.6-0.9,3.1-1.3,4.7C82.3,59.1,76.9,58.3,71.4,57.6z"/></g></svg>

AppImageBuilder.yml

@@ -0,0 +1,73 @@
+version: 1
+AppDir:
+  path: ./AppDir
+  app_info:
+    id: io.filecoin.lotus
+    name: Lotus
+    icon: icon
+    version: latest
+    exec: usr/bin/lotus
+    exec_args: $@
+  apt:
+    arch: amd64
+    allow_unauthenticated: true
+    sources:
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal main restricted
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal universe
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates universe
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal multiverse
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-updates multiverse
+    - sourceline: deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
+        universe multiverse
+    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security main restricted
+    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security universe
+    - sourceline: deb http://security.ubuntu.com/ubuntu focal-security multiverse
+    - sourceline: deb https://cli-assets.heroku.com/apt ./
+    - sourceline: deb http://ppa.launchpad.net/openjdk-r/ppa/ubuntu focal main
+    - sourceline: deb http://ppa.launchpad.net/git-core/ppa/ubuntu focal main
+    - sourceline: deb http://archive.canonical.com/ubuntu focal partner
+    include:
+    - ocl-icd-libopencl1
+    - libhwloc15
+    exclude: []
+  files:
+    include:
+    - /usr/lib/x86_64-linux-gnu/libgcc_s.so.1
+    - /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
+    - /usr/lib/x86_64-linux-gnu/libm-2.31.so
+    - /usr/lib/x86_64-linux-gnu/libdl-2.31.so
+    - /usr/lib/x86_64-linux-gnu/libc-2.31.so
+    - /usr/lib/x86_64-linux-gnu/libudev.so.1.6.17
+    exclude:
+    - usr/share/man
+    - usr/share/doc/*/README.*
+    - usr/share/doc/*/changelog.*
+    - usr/share/doc/*/NEWS.*
+    - usr/share/doc/*/TODO.*
+  test:
+    fedora:
+      image: appimagecrafters/tests-env:fedora-30
+      command: ./AppRun
+      use_host_x: false
+    debian:
+      image: appimagecrafters/tests-env:debian-stable
+      command: ./AppRun
+      use_host_x: false
+    arch:
+      image: appimagecrafters/tests-env:archlinux-latest
+      command: ./AppRun
+      use_host_x: false
+    centos:
+      image: appimagecrafters/tests-env:centos-7
+      command: ./AppRun
+      use_host_x: false
+    ubuntu:
+      image: appimagecrafters/tests-env:ubuntu-xenial
+      command: ./AppRun
+      use_host_x: false
+AppImage:
+  arch: x86_64
+  update-information: guess
+  sign-key: None
+

Dockerfile

@@ -1,140 +0,0 @@
-#####################################
-FROM golang:1.20.7-bullseye AS lotus-builder
-MAINTAINER Lotus Development Team
-
-RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
-
-ENV XDG_CACHE_HOME="/tmp"
-
-### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.63.0
-
-RUN set -eux; \
-    dpkgArch="$(dpkg --print-architecture)"; \
-    case "${dpkgArch##*-}" in \
-        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
-        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
-        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
-    esac; \
-    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
-    wget "$url"; \
-    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
-
-COPY ./ /opt/filecoin
-WORKDIR /opt/filecoin
-
-RUN scripts/docker-git-state-check.sh
-
-### make configurable filecoin-ffi build
-ARG FFI_BUILD_FROM_SOURCE=0
-ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-
-RUN make clean deps
-
-ARG RUSTFLAGS=""
-ARG GOFLAGS=""
-
-RUN make buildall
-
-#####################################
-FROM ubuntu:20.04 AS lotus-base
-MAINTAINER Lotus Development Team
-
-# Base resources
-COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
-COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
-COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
-COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
-COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
-COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
-COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
-COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
-COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
-
-RUN useradd -r -u 532 -U fc \
- && mkdir -p /etc/OpenCL/vendors \
- && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
-
-#####################################
-FROM lotus-base AS lotus
-MAINTAINER Lotus Development Team
-
-COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
-COPY scripts/docker-lotus-entrypoint.sh /
-
-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_PATH /var/lib/lotus
-ENV DOCKER_LOTUS_IMPORT_WALLET ""
-
-RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
-
-VOLUME /var/lib/lotus
-VOLUME /var/tmp/filecoin-proof-parameters
-
-USER fc
-
-EXPOSE 1234
-
-ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
-
-CMD ["-help"]
-
-#####################################
-FROM lotus-base AS lotus-all-in-one
-
-ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
-ENV LOTUS_MINER_PATH /var/lib/lotus-miner
-ENV LOTUS_PATH /var/lib/lotus
-ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
-ENV WALLET_PATH /var/lib/lotus-wallet
-
-COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-seed     /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-wallet   /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-gateway  /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-miner    /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-worker   /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-provider    /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
-COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
-
-RUN mkdir /var/tmp/filecoin-proof-parameters
-RUN mkdir /var/lib/lotus
-RUN mkdir /var/lib/lotus-miner
-RUN mkdir /var/lib/lotus-worker
-RUN mkdir /var/lib/lotus-wallet
-RUN mkdir /var/lib/lotus-provider
-RUN chown fc: /var/tmp/filecoin-proof-parameters
-RUN chown fc: /var/lib/lotus
-RUN chown fc: /var/lib/lotus-miner
-RUN chown fc: /var/lib/lotus-worker
-RUN chown fc: /var/lib/lotus-wallet
-RUN chown fc: /var/lib/lotus-provider
-
-
-VOLUME /var/tmp/filecoin-proof-parameters
-VOLUME /var/lib/lotus
-VOLUME /var/lib/lotus-miner
-VOLUME /var/lib/lotus-worker
-VOLUME /var/lib/lotus-wallet
-VOLUME /var/lib/lotus-provider
-
-EXPOSE 1234
-EXPOSE 2345
-EXPOSE 3456
-EXPOSE 1777

Dockerfile.lotus

@@ -0,0 +1,256 @@
+FROM golang:1.17.9-buster AS builder-deps
+MAINTAINER Lotus Development Team
+
+RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
+
+ARG RUST_VERSION=nightly
+ENV XDG_CACHE_HOME="/tmp"
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+
+FROM builder-deps AS builder-local
+MAINTAINER Lotus Development Team
+
+COPY ./ /opt/filecoin
+WORKDIR /opt/filecoin
+RUN make clean deps
+
+
+FROM builder-local AS builder-test
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+RUN make debug
+
+
+FROM builder-local AS builder
+MAINTAINER Lotus Development Team
+
+WORKDIR /opt/filecoin
+
+ARG RUSTFLAGS=""
+ARG GOFLAGS=""
+
+RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats
+
+
+FROM ubuntu:20.04 AS base
+MAINTAINER Lotus Development Team
+
+# Base resources
+COPY --from=builder /etc/ssl/certs                           /etc/ssl/certs
+COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2         /lib/
+COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1         /lib/
+COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1      /lib/
+COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1       /lib/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7   /lib/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
+
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+###
+FROM base AS lotus
+MAINTAINER Lotus Development Team
+
+COPY --from=builder /opt/filecoin/lotus /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
+COPY scripts/docker-lotus-entrypoint.sh /
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
+ENV DOCKER_LOTUS_IMPORT_WALLET ""
+
+RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
+
+VOLUME /var/lib/lotus
+VOLUME /var/tmp/filecoin-proof-parameters
+
+USER fc
+
+EXPOSE 1234
+
+ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
+
+CMD ["-help"]
+
+###
+FROM base AS lotus-wallet
+MAINTAINER Lotus Development Team
+
+COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
+
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/lib/lotus-wallet
+
+VOLUME /var/lib/lotus-wallet
+
+USER fc
+
+EXPOSE 1777
+
+ENTRYPOINT ["/usr/local/bin/lotus-wallet"]
+
+CMD ["-help"]
+
+###
+FROM base AS lotus-gateway
+MAINTAINER Lotus Development Team
+
+COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
+
+USER fc
+
+EXPOSE 1234
+
+ENTRYPOINT ["/usr/local/bin/lotus-gateway"]
+
+CMD ["-help"]
+
+
+###
+FROM base AS lotus-miner
+MAINTAINER Lotus Development Team
+
+COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
+COPY scripts/docker-lotus-miner-entrypoint.sh /
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+
+RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters
+
+VOLUME /var/lib/lotus-miner
+VOLUME /var/tmp/filecoin-proof-parameters
+
+USER fc
+
+EXPOSE 2345
+
+ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"]
+
+CMD ["-help"]
+
+
+###
+FROM base AS lotus-worker
+MAINTAINER Lotus Development Team
+
+COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+
+RUN mkdir /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-worker
+
+VOLUME /var/lib/lotus-worker
+
+USER fc
+
+EXPOSE 3456
+
+ENTRYPOINT ["/usr/local/bin/lotus-worker"]
+
+CMD ["-help"]
+
+
+###
+from base as lotus-all-in-one
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
+
+COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder /opt/filecoin/lotus-stats /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+
+###
+from base as lotus-test
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=builder-test /opt/filecoin/lotus      /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-miner /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-worker /usr/local/bin/
+COPY --from=builder-test /opt/filecoin/lotus-seed /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777
+

GO_VERSION_MIN

@@ -1 +0,0 @@
-1.20.7

LOTUS_RELEASE_FLOW.md

@@ -73,7 +73,7 @@ All releases under an odd minor version number indicate **feature releases**. Th
 
 Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes.  There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
 
-We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the [Release Cycle section](#release-cycle).
+We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the Release Cycle section (TODO: Link).
 
 ### Examples Scenarios
 

Makefile

@@ -8,11 +8,9 @@ unexport GOFLAGS
 GOCC?=go
 
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
-GOVERSIONMIN:=$(shell cat GO_VERSION_MIN | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
+ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
-
-ifeq ($(shell expr $(GOVERSION) \< $(GOVERSIONMIN)), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
-$(error Update Golang to version to at least $(shell cat GO_VERSION_MIN))
+$(error Update Golang to version to at least 1.17.9)
 endif
 
 # git modules that need to be loaded
@@ -66,7 +64,7 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-build-devnets: build lotus-seed lotus-shed lotus-provider
+build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
 .PHONY: build-devnets
 
 debug: GOFLAGS+=-tags=debug
@@ -97,15 +95,6 @@ lotus-miner: $(BUILD_DEPS)
 .PHONY: lotus-miner
 BINS+=lotus-miner
 
-lotus-provider: $(BUILD_DEPS)
-	rm -f lotus-provider
-	$(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider
-.PHONY: lotus-provider
-BINS+=lotus-provider
-
-lp2k: GOFLAGS+=-tags=2k
-lp2k: lotus-provider
-
 lotus-worker: $(BUILD_DEPS)
 	rm -f lotus-worker
 	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
@@ -130,7 +119,7 @@ an existing lotus binary in your PATH. This may cause problems if you don't run
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker install-provider
+install: install-daemon install-miner install-worker
 
 install-daemon:
 	install -C ./lotus /usr/local/bin/lotus
@@ -138,30 +127,12 @@ install-daemon:
 install-miner:
 	install -C ./lotus-miner /usr/local/bin/lotus-miner
 
-install-provider:
-	install -C ./lotus-provider /usr/local/bin/lotus-provider
-
 install-worker:
 	install -C ./lotus-worker /usr/local/bin/lotus-worker
 
 install-app:
 	install -C ./$(APP) /usr/local/bin/$(APP)
 
-uninstall: uninstall-daemon uninstall-miner uninstall-worker
-.PHONY: uninstall
-
-uninstall-daemon:
-	rm -f /usr/local/bin/lotus
-
-uninstall-miner:
-	rm -f /usr/local/bin/lotus-miner
-
-uninstall-provider:
-	rm -f /usr/local/bin/lotus-provider
-
-uninstall-worker:
-	rm -f /usr/local/bin/lotus-worker
-
 # TOOLS
 
 lotus-seed: $(BUILD_DEPS)
@@ -177,16 +148,28 @@ benchmarks:
 	@curl -X POST 'http://benchmark.kittyhawk.wtf/benchmark' -d '@bench.json' -u "${benchmark_http_cred}"
 .PHONY: benchmarks
 
+lotus-pond: 2k
+	$(GOCC) build -o lotus-pond ./lotuspond
+.PHONY: lotus-pond
+BINS+=lotus-pond
+
+lotus-pond-front:
+	(cd lotuspond/front && npm i && CI=false npm run build)
+.PHONY: lotus-pond-front
+
+lotus-pond-app: lotus-pond-front lotus-pond
+.PHONY: lotus-pond-app
+
 lotus-fountain:
 	rm -f lotus-fountain
-	$(GOCC) build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
+	go build $(GOFLAGS) -o lotus-fountain ./cmd/lotus-fountain
-	$(GOCC) run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-fountain -i ./cmd/lotus-fountain -i ./build
 .PHONY: lotus-fountain
 BINS+=lotus-fountain
 
 lotus-bench:
 	rm -f lotus-bench
-	$(GOCC) build $(GOFLAGS) -o lotus-bench ./cmd/lotus-bench
+	$(GOCC) build -o lotus-bench ./cmd/lotus-bench
 .PHONY: lotus-bench
 BINS+=lotus-bench
 
@@ -208,9 +191,9 @@ lotus-health:
 .PHONY: lotus-health
 BINS+=lotus-health
 
-lotus-wallet: $(BUILD_DEPS)
+lotus-wallet:
 	rm -f lotus-wallet
-	$(GOCC) build $(GOFLAGS) -o lotus-wallet ./cmd/lotus-wallet
+	$(GOCC) build -o lotus-wallet ./cmd/lotus-wallet
 .PHONY: lotus-wallet
 BINS+=lotus-wallet
 
@@ -256,14 +239,6 @@ install-miner-service: install-miner install-daemon-service
 	@echo
 	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
 
-install-provider-service: install-provider install-daemon-service
-	mkdir -p /etc/systemd/system
-	mkdir -p /var/log/lotus
-	install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service
-	systemctl daemon-reload
-	@echo
-	@echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup."
-
 install-main-services: install-miner-service
 
 install-all-services: install-main-services
@@ -282,12 +257,6 @@ clean-miner-service:
 	rm -f /etc/systemd/system/lotus-miner.service
 	systemctl daemon-reload
 
-clean-provider-service:
-	-systemctl stop lotus-provider
-	-systemctl disable lotus-provider
-	rm -f /etc/systemd/system/lotus-provider.service
-	systemctl daemon-reload
-
 clean-main-services: clean-daemon-service
 
 clean-all-services: clean-main-services
@@ -318,21 +287,14 @@ type-gen: api-gen
 	$(GOCC) generate -x ./...
 	goimports -w api/
 
-actors-code-gen:
+method-gen: api-gen
+	(cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go)
+
+actors-gen:
 	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
-actors-gen: actors-code-gen 
-	./scripts/fiximports
-.PHONY: actors-gen
-
-bundle-gen:
-	$(GOCC) run ./gen/bundle $(VERSION) $(RELEASE) $(RELEASE_OVERRIDES)
-	$(GOCC) fmt ./build/...
-.PHONY: bundle-gen
-
-
 api-gen:
 	$(GOCC) run ./gen/api
 	goimports -w api
@@ -351,14 +313,14 @@ appimage: lotus
 	cp ./lotus AppDir/usr/bin/
 	appimage-builder
 
-docsgen: docsgen-md docsgen-openrpc fiximports
+docsgen: docsgen-md docsgen-openrpc
 
 docsgen-md-bin: api-gen actors-gen
 	$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
 docsgen-openrpc-bin: api-gen actors-gen
 	$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
 
-docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker docsgen-md-provider
+docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
 
 docsgen-md-full: docsgen-md-bin
 	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
@@ -367,10 +329,8 @@ docsgen-md-storage: docsgen-md-bin
 	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
 docsgen-md-worker: docsgen-md-bin
 	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
-docsgen-md-provider: docsgen-md-bin
-	./docgen-md "api/api_lp.go" "Provider" "api" "./api" > documentation/en/api-v0-methods-provider.md
 
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
+docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
 
 docsgen-openrpc-full: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
@@ -378,31 +338,24 @@ docsgen-openrpc-storage: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
 docsgen-openrpc-worker: docsgen-openrpc-bin
 	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
-docsgen-openrpc-gateway: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" -gzip > build/openrpc/gateway.json.gz
 
 .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
 
-fiximports:
+gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
-	./scripts/fiximports
+	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
-
-gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci
-	./scripts/fiximports
-	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'"
 .PHONY: gen
 
 jen: gen
 
-snap: lotus lotus-miner lotus-worker lotus-provider
+snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
 
 # separate from gen because it needs binaries
-docsgen-cli: lotus lotus-miner lotus-worker lotus-provider
+docsgen-cli: lotus lotus-miner lotus-worker
-	python3 ./scripts/generate-lotus-cli.py
+	python ./scripts/generate-lotus-cli.py
 	./lotus config default > documentation/en/default-lotus-config.toml
 	./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
-	./lotus-provider config default > documentation/en/default-lotus-provider-config.toml
 .PHONY: docsgen-cli
 
 print-%:

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <a href="https://lotus.filecoin.io/" title="Filecoin Docs">
+  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
     <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
   </a>
 </p>
@@ -10,7 +10,7 @@
   <a href="https://circleci.com/gh/filecoin-project/lotus"><img src="https://circleci.com/gh/filecoin-project/lotus.svg?style=svg"></a>
   <a href="https://codecov.io/gh/filecoin-project/lotus"><img src="https://codecov.io/gh/filecoin-project/lotus/branch/master/graph/badge.svg"></a>
   <a href="https://goreportcard.com/report/github.com/filecoin-project/lotus"><img src="https://goreportcard.com/badge/github.com/filecoin-project/lotus" /></a>  
-  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.18.8-blue.svg" /></a>
+  <a href=""><img src="https://img.shields.io/badge/golang-%3E%3D1.16-blue.svg" /></a>
   <br>
 </p>
 
@@ -20,7 +20,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
 
 > Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases).
  
-For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
+For complete instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/). Basic build instructions can be found further down in this readme.
 
 ## Reporting a Vulnerability
 
@@ -31,7 +31,7 @@ Please send an email to security@filecoin.org. See our [security policy](SECURIT
 These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
 
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [builtin-actors](https://github.com/filecoin-project/builtin-actors)
+- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
 
 ## Contribute
 
@@ -64,17 +64,17 @@ sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config cu
 
 Fedora:
 ```
-sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc hwloc-devel
+sudo dnf -y install gcc make git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm wget hwloc libhwloc-dev
 ```
 
-For other distributions you can find the required dependencies [here.](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms) For instructions specific to macOS, you can find them [here.](https://lotus.filecoin.io/lotus/install/macos/)
+For other distributions you can find the required dependencies [here.](https://docs.filecoin.io/get-started/lotus/installation/#system-specific) For instructions specific to macOS, you can find them [here.](https://docs.filecoin.io/get-started/lotus/installation/#macos)
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.20.10 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.17.9 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.20.10.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.17.9.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**
@@ -101,7 +101,7 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
 2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases).
 
-   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://lotus.filecoin.io/lotus/manage/switch-networks/) before proceeding.
+   If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://docs.filecoin.io/get-started/lotus/switch-networks/) before proceeding.
 
    For networks other than mainnet, look up the current branch or tag/commit for the network you want to join in the [Filecoin networks dashboard](https://network.filecoin.io), then build Lotus for your specific network below.
 
@@ -113,8 +113,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
    Currently, the latest code on the _master_ branch corresponds to mainnet.
 
-3. If you are in China, see "[Lotus: tips when running in China](https://lotus.filecoin.io/lotus/configure/nodes-in-china/)".
+3. If you are in China, see "[Lotus: tips when running in China](https://docs.filecoin.io/get-started/lotus/tips-running-in-china/)".
-4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://lotus.filecoin.io/lotus/install/prerequisites/). Note, if you are building the proof binaries from source, [installing rustup](https://lotus.filecoin.io/lotus/install/linux/#rustup) is also needed.
+4. This build instruction uses the prebuilt proofs binaries. If you want to build the proof binaries from source check the [complete instructions](https://docs.filecoin.io/get-started/lotus/installation/#build-and-install-lotus). Note, if you are building the proof binaries from source, [installing rustup](https://docs.filecoin.io/get-started/lotus/installation/#rustup) is also needed.
 
 5. Build and install Lotus:
 
@@ -129,11 +129,9 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
    This will put `lotus`, `lotus-miner` and `lotus-worker` in `/usr/local/bin`.
 
-   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://lotus.filecoin.io/lotus/configure/defaults/#environment-variables) for information on how to customize the Lotus folder.
+   `lotus` will use the `$HOME/.lotus` folder by default for storage (configuration, chain data, wallets, etc). See [advanced options](https://docs.filecoin.io/get-started/lotus/configuration-and-advanced-usage/) for information on how to customize the Lotus folder.
 
-6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
+6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://docs.filecoin.io/get-started/lotus/installation/#start-the-lotus-daemon-and-sync-the-chain).
-
-7. (Optional) Follow the [Setting Up Prometheus and Grafana](https://github.com/filecoin-project/lotus/tree/master/metrics/README.md) guide for detailed instructions on setting up a working monitoring system running against a local running lotus node.
 
 ## License
 

api/api_common.go

@@ -3,14 +3,13 @@ package api
 import (
 	"context"
 	"fmt"
-	"time"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/journal/alerting"
 
 	"github.com/google/uuid"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
-
-	apitypes "github.com/filecoin-project/lotus/api/types"
-	"github.com/filecoin-project/lotus/journal/alerting"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -50,9 +49,6 @@ type Common interface {
 	// trigger graceful shutdown
 	Shutdown(context.Context) error //perm:admin
 
-	// StartTime returns node start time
-	StartTime(context.Context) (time.Time, error) //perm:read
-
 	// Session returns a random UUID of api provider session
 	Session(context.Context) (uuid.UUID, error) //perm:read
 

api/api_errors.go

@@ -1,42 +0,0 @@
-package api
-
-import (
-	"errors"
-	"reflect"
-
-	"github.com/filecoin-project/go-jsonrpc"
-)
-
-const (
-	EOutOfGas = iota + jsonrpc.FirstUserCode
-	EActorNotFound
-)
-
-type ErrOutOfGas struct{}
-
-func (e *ErrOutOfGas) Error() string {
-	return "call ran out of gas"
-}
-
-type ErrActorNotFound struct{}
-
-func (e *ErrActorNotFound) Error() string {
-	return "actor not found"
-}
-
-var RPCErrors = jsonrpc.NewErrors()
-
-func ErrorIsIn(err error, errorTypes []error) bool {
-	for _, etype := range errorTypes {
-		tmp := reflect.New(reflect.PointerTo(reflect.ValueOf(etype).Elem().Type())).Interface()
-		if errors.As(err, tmp) {
-			return true
-		}
-	}
-	return false
-}
-
-func init() {
-	RPCErrors.Register(EOutOfGas, new(*ErrOutOfGas))
-	RPCErrors.Register(EActorNotFound, new(*ErrActorNotFound))
-}

api/api_full.go

@@ -6,32 +6,27 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/google/uuid"
-	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
-	"github.com/filecoin-project/go-state-types/builtin/v9/market"
-	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
-	abinetwork "github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
 
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 )
@@ -42,7 +37,6 @@ import (
 type ChainIO interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainPutObj(context.Context, blocks.Block) error
 }
 
 const LookbackNoLimit = abi.ChainEpoch(-1)
@@ -128,9 +122,6 @@ type FullNode interface {
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
 	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
 
-	// ChainPutObj puts a given object into the block store
-	ChainPutObj(context.Context, blocks.Block) error //perm:admin
-
 	// ChainStatObj returns statistics about the graph referenced by 'obj'.
 	// If 'base' is also specified, then the returned stat will be a diff
 	// between the two objects.
@@ -152,7 +143,7 @@ type FullNode interface {
 
 	// ChainGetPath returns a set of revert/apply operations needed to get from
 	// one tipset to another, for example:
-	// ```
+	//```
 	//        to
 	//         ^
 	// from   tAA
@@ -161,7 +152,7 @@ type FullNode interface {
 	//  ^---*--^
 	//      ^
 	//     tRR
-	// ```
+	//```
 	// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) //perm:read
 
@@ -172,24 +163,6 @@ type FullNode interface {
 	// If oldmsgskip is set, messages from before the requested roots are also not included.
 	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
 
-	// ChainExportRangeInternal triggers the export of a chain
-	// CAR-snapshot directly to disk. It is similar to ChainExport,
-	// except, depending on options, the snapshot can include receipts,
-	// messages and stateroots for the length between the specified head
-	// and tail, thus producing "archival-grade" snapshots that include
-	// all the on-chain data.  The header chain is included back to
-	// genesis and these snapshots can be used to initialize Filecoin
-	// nodes.
-	ChainExportRangeInternal(ctx context.Context, head, tail types.TipSetKey, cfg ChainExportConfig) error //perm:admin
-
-	// ChainPrune forces compaction on cold store and garbage collects; only supported if you
-	// are using the splitstore
-	ChainPrune(ctx context.Context, opts PruneOpts) error //perm:admin
-
-	// ChainHotGC does online (badger) GC on the hot store; only supported if you are using
-	// the splitstore
-	ChainHotGC(ctx context.Context, opts HotGCOpts) error //perm:admin
-
 	// ChainCheckBlockstore performs an (asynchronous) health check on the chain/state blockstore
 	// if supported by the underlying implementation.
 	ChainCheckBlockstore(context.Context) error //perm:admin
@@ -197,8 +170,13 @@ type FullNode interface {
 	// ChainBlockstoreInfo returns some basic information about the blockstore
 	ChainBlockstoreInfo(context.Context) (map[string]interface{}, error) //perm:read
 
-	// ChainGetEvents returns the events under an event AMT root CID.
+	// MethodGroup: Beacon
-	ChainGetEvents(context.Context, cid.Cid) ([]types.Event, error) //perm:read
+	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
+
+	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
+	// the entry has not yet been produced, the call will block until the entry
+	// becomes available
+	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
 
 	// GasEstimateFeeCap estimates gas fee cap
 	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
@@ -296,10 +274,8 @@ type FullNode interface {
 	MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
 	MpoolSub(context.Context) (<-chan MpoolUpdate, error)           //perm:read
 
-	// MpoolClear clears pending messages from the mpool.
+	// MpoolClear clears pending messages from the mpool
-	// If clearLocal is true, ALL messages will be cleared.
+	MpoolClear(context.Context, bool) error //perm:write
-	// If clearLocal is false, local messages will be protected, all others will be cleared.
-	MpoolClear(ctx context.Context, clearLocal bool) error //perm:write
 
 	// MpoolGetConfig returns (a copy of) the current mpool config
 	MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read
@@ -408,12 +384,12 @@ type FullNode interface {
 	ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
 
 	// ClientUnimport removes references to the specified file from filestore
-	// ClientUnimport(path string)
+	//ClientUnimport(path string)
 
 	// ClientListImports lists imported files and their root CIDs
 	ClientListImports(ctx context.Context) ([]Import, error) //perm:write
 
-	// ClientListAsks() []Ask
+	//ClientListAsks() []Ask
 
 	// MethodGroup: State
 	// The State methods are used to query, inspect, and interact with chain state.
@@ -428,7 +404,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is not found on chain,
+	// If a tipset key is provided, and a replacing message is found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -467,7 +443,7 @@ type FullNode interface {
 	// StateMinerPower returns the power of the indicated miner
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
 	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (MinerInfo, error) //perm:read
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
 	// StateMinerDeadlines returns all the proving deadlines for the given miner
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
 	// StateMinerPartitions returns all partitions in the specified deadline
@@ -484,15 +460,10 @@ type FullNode interface {
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
-	// StateMinerSectorAllocated checks if a sector number is marked as allocated.
+	// StateMinerSectorAllocated checks if a sector is allocated
 	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
-	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector.
+	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	// Returns nil and no error if the sector isn't precommitted.
+	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
-	//
-	// Note that the sector number may be allocated while PreCommitInfo is nil. This means that either allocated sector
-	// numbers were compacted, and the sector number was marked as allocated in order to reduce size of the allocated
-	// sectors bitfield, or that the sector was precommitted, but the precommit has expired.
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorPreCommitOnChainInfo, error) //perm:read
 	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
@@ -546,35 +517,18 @@ type FullNode interface {
 	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
 	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
 	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]*MarketDeal, error) //perm:read
+	StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
-	// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal. Returns nil if
-	// pending allocation is not found.
-	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
-	// StateGetAllocation returns the allocation for a given address and allocation ID.
-	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
-	// StateGetAllocations returns the all the allocations for a given client.
-	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
-	// StateGetClaim returns the claim for a given address and claim ID.
-	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
-	// StateGetClaims returns the all the claims for a given provider.
-	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
-	// StateComputeDataCID computes DataCID from a set of on-chain deals
-	StateComputeDataCID(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tsk types.TipSetKey) (cid.Cid, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
 	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateAccountKey returns the public key address of the given ID address for secp and bls accounts
+	// StateAccountKey returns the public key address of the given ID address
 	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
-	// StateLookupRobustAddress returns the public key address of the given ID address for non-account addresses (multisig, miners etc)
-	StateLookupRobustAddress(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateChangedActors returns all the actors whose states change between the two given state CIDs
 	// TODO: Should this take tipset keys instead?
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
 	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
 	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
-	// StateMinerAllocated returns a bitfield containing all sector numbers marked as allocated in miner state
-	StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) //perm:read
 	// StateCompute is a flexible command that applies the given messages on the given tipset.
 	// The messages are run as though the VM were at the provided height.
 	//
@@ -616,7 +570,7 @@ type FullNode interface {
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
+	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
 	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
 	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
 	// can issue. It takes the deal size and verified status as parameters.
@@ -630,29 +584,12 @@ type FullNode interface {
 	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
 	// StateNetworkVersion returns the network version at the given tipset
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
-	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
-	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
-	// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
-	StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 	// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
 	StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 
-	// StateGetRandomnessDigestFromTickets. is used to sample the chain for randomness.
-	StateGetRandomnessDigestFromTickets(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
-	// StateGetRandomnessDigestFromBeacon is used to sample the beacon for randomness.
-	StateGetRandomnessDigestFromBeacon(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
-
-	// StateGetBeaconEntry returns the beacon entry for the given filecoin epoch. If
-	// the entry has not yet been produced, the call will block until the entry
-	// becomes available
-	StateGetBeaconEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
-
-	// StateGetNetworkParams return current network params
-	StateGetNetworkParams(ctx context.Context) (*NetworkParams, error) //perm:read
-
 	// MethodGroup: Msig
 	// The Msig methods are used to interact with multisig wallets on the
 	// filecoin network
@@ -665,14 +602,14 @@ type FullNode interface {
 	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
 	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
 
-	// MsigGetPending returns pending transactions for the given multisig
+	//MsigGetPending returns pending transactions for the given multisig
-	// wallet. Once pending transactions are fully approved, they will no longer
+	//wallet. Once pending transactions are fully approved, they will no longer
-	// appear here.
+	//appear here.
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) //perm:read
 
 	// MsigCreate creates a multisig wallet
 	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
-	// <initial balance>, <sender address of the create msg>, <gas price>
+	//<initial balance>, <sender address of the create msg>, <gas price>
 	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (*MessagePrototype, error) //perm:sign
 
 	// MsigPropose proposes a multisig message
@@ -784,96 +721,6 @@ type FullNode interface {
 
 	NodeStatus(ctx context.Context, inclChainStatus bool) (NodeStatus, error) //perm:read
 
-	// MethodGroup: Eth
-	// These methods are used for Ethereum-compatible JSON-RPC calls
-	//
-	// EthAccounts will always return [] since we don't expect Lotus to manage private keys
-	EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) //perm:read
-	// EthAddressToFilecoinAddress converts an EthAddress into an f410 Filecoin Address
-	EthAddressToFilecoinAddress(ctx context.Context, ethAddress ethtypes.EthAddress) (address.Address, error) //perm:read
-	// FilecoinAddressToEthAddress converts an f410 or f0 Filecoin Address to an EthAddress
-	FilecoinAddressToEthAddress(ctx context.Context, filecoinAddress address.Address) (ethtypes.EthAddress, error) //perm:read
-	// EthBlockNumber returns the height of the latest (heaviest) TipSet
-	EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error) //perm:read
-	// EthGetBlockTransactionCountByNumber returns the number of messages in the TipSet
-	EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error) //perm:read
-	// EthGetBlockTransactionCountByHash returns the number of messages in the TipSet
-	EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error) //perm:read
-
-	EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error)                                //perm:read
-	EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error)                                         //perm:read
-	EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error)                                             //perm:read
-	EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error)                //perm:read
-	EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error)                                                     //perm:read
-	EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error)                                          //perm:read
-	EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthUint64, error) //perm:read
-	EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error)                                               //perm:read
-	EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error)                  //perm:read
-	EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash ethtypes.EthHash, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)     //perm:read
-	EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum ethtypes.EthUint64, txIndex ethtypes.EthUint64) (ethtypes.EthTx, error)  //perm:read
-
-	EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)                                  //perm:read
-	EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
-	EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBigInt, error)                              //perm:read
-	EthChainId(ctx context.Context) (ethtypes.EthUint64, error)                                                                                                      //perm:read
-	EthSyncing(ctx context.Context) (ethtypes.EthSyncingResult, error)                                                                                               //perm:read
-	NetVersion(ctx context.Context) (string, error)                                                                                                                  //perm:read
-	NetListening(ctx context.Context) (bool, error)                                                                                                                  //perm:read
-	EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error)                                                                                              //perm:read
-	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)                                                                                                     //perm:read
-	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)                                                                          //perm:read
-
-	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)                                             //perm:read
-	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)                                 //perm:read
-	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
-
-	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read
-
-	// Returns event logs matching given filter spec.
-	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error) //perm:read
-
-	// Polling method for a filter, returns event logs which occurred since last poll.
-	// (requires write perm since timestamp of last filter execution will be written)
-	EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:read
-
-	// Returns event logs matching filter with given id.
-	// (requires write perm since timestamp of last filter execution will be written)
-	EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error) //perm:read
-
-	// Installs a persistent filter based on given filter spec.
-	EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error) //perm:read
-
-	// Installs a persistent filter to notify when a new block arrives.
-	EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:read
-
-	// Installs a persistent filter to notify when new messages arrive in the message pool.
-	EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error) //perm:read
-
-	// Uninstalls a filter with given id.
-	EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error) //perm:read
-
-	// Subscribe to different event types using websockets
-	// eventTypes is one or more of:
-	//  - newHeads: notify when new blocks arrive.
-	//  - pendingTransactions: notify when new messages arrive in the message pool.
-	//  - logs: notify new event logs that match a criteria
-	// params contains additional parameters used with the log event type
-	// The client will receive a stream of EthSubscriptionResponse values until EthUnsubscribe is called.
-	EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error) //perm:read
-
-	// Unsubscribe from a websocket subscription
-	EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error) //perm:read
-
-	// Returns the client version
-	Web3ClientVersion(ctx context.Context) (string, error) //perm:read
-
-	// TraceAPI related methods
-	//
-	// Returns traces created at given block
-	EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error) //perm:read
-	// Replays all transactions in a block returning the requested traces for each transaction
-	EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) //perm:read
-
 	// CreateBackup creates node backup onder the specified file name. The
 	// method requires that the lotus daemon is running with the
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
@@ -881,12 +728,6 @@ type FullNode interface {
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 }
 
-// reverse interface to the client, called after EthSubscribe
-type EthSubscriber interface {
-	// note: the parameter is ethtypes.EthSubscriptionResponse serialized as json object
-	EthSubscription(ctx context.Context, r jsonrpc.RawParams) error // rpc_method:eth_subscription notify:true
-}
-
 type StorageAsk struct {
 	Response *storagemarket.StorageAsk
 
@@ -1134,12 +975,8 @@ type RetrievalOrder struct {
 	Client                  address.Address
 	Miner                   address.Address
 	MinerPeer               *retrievalmarket.RetrievalPeer
-
-	RemoteStore *RemoteStoreID `json:"RemoteStore,omitempty"`
 }
 
-type RemoteStoreID = uuid.UUID
-
 type InvocResult struct {
 	MsgCid         cid.Cid
 	Msg            *types.Message
@@ -1365,32 +1202,3 @@ type MsigTransaction struct {
 
 	Approved []address.Address
 }
-
-type PruneOpts struct {
-	MovingGC    bool
-	RetainState int64
-}
-
-type HotGCOpts struct {
-	Threshold float64
-	Periodic  bool
-	Moving    bool
-}
-
-type EthTxReceipt struct {
-	TransactionHash   ethtypes.EthHash     `json:"transactionHash"`
-	TransactionIndex  ethtypes.EthUint64   `json:"transactionIndex"`
-	BlockHash         ethtypes.EthHash     `json:"blockHash"`
-	BlockNumber       ethtypes.EthUint64   `json:"blockNumber"`
-	From              ethtypes.EthAddress  `json:"from"`
-	To                *ethtypes.EthAddress `json:"to"`
-	StateRoot         ethtypes.EthHash     `json:"root"`
-	Status            ethtypes.EthUint64   `json:"status"`
-	ContractAddress   *ethtypes.EthAddress `json:"contractAddress"`
-	CumulativeGasUsed ethtypes.EthUint64   `json:"cumulativeGasUsed"`
-	GasUsed           ethtypes.EthUint64   `json:"gasUsed"`
-	EffectiveGasPrice ethtypes.EthBigInt   `json:"effectiveGasPrice"`
-	LogsBloom         ethtypes.EthBytes    `json:"logsBloom"`
-	Logs              []ethtypes.EthLog    `json:"logs"`
-	Type              ethtypes.EthUint64   `json:"type"`
-}

api/api_gateway.go

@@ -3,20 +3,15 @@ package api
 import (
 	"context"
 
-	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
-	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/dline"
 
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/ethtypes"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -27,21 +22,14 @@ import (
 // When adding / changing methods in this file:
 // * Do the change here
 // * Adjust implementation in `node/impl/`
-// * Run `make clean && make deps && make gen` - this will:
+// * Run `make gen` - this will:
 //  * Generate proxy structs
 //  * Generate mocks
 //  * Generate markdown docs
 //  * Generate openrpc blobs
 
 type Gateway interface {
-	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
-	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
-	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*MiningBaseInfo, error)
-	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error)
-	GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
-	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)
 	ChainGetParentMessages(context.Context, cid.Cid) ([]Message, error)
 	ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)
@@ -55,78 +43,27 @@ type Gateway interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainGetGenesis(context.Context) (*types.TipSet, error)
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
 	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*InvocResult, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
-	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
-	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
-	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
-	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
-	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
 	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (MinerInfo, error)
+	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
-	StateNetworkName(context.Context) (dtypes.NetworkName, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
-	StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
 	StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
 	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
 	Version(context.Context) (APIVersion, error)
-	Discover(context.Context) (apitypes.OpenRPCDocument, error)
-
-	EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error)
-	EthBlockNumber(ctx context.Context) (ethtypes.EthUint64, error)
-	EthGetBlockTransactionCountByNumber(ctx context.Context, blkNum ethtypes.EthUint64) (ethtypes.EthUint64, error)
-	EthGetBlockTransactionCountByHash(ctx context.Context, blkHash ethtypes.EthHash) (ethtypes.EthUint64, error)
-	EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthHash, fullTxInfo bool) (ethtypes.EthBlock, error)
-	EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (ethtypes.EthBlock, error)
-	EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error)
-	EthGetTransactionByHashLimited(ctx context.Context, txHash *ethtypes.EthHash, limit abi.ChainEpoch) (*ethtypes.EthTx, error)
-	EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*ethtypes.EthHash, error)
-	EthGetMessageCidByTransactionHash(ctx context.Context, txHash *ethtypes.EthHash) (*cid.Cid, error)
-	EthGetTransactionCount(ctx context.Context, sender ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthUint64, error)
-	EthGetTransactionReceipt(ctx context.Context, txHash ethtypes.EthHash) (*EthTxReceipt, error)
-	EthGetTransactionReceiptLimited(ctx context.Context, txHash ethtypes.EthHash, limit abi.ChainEpoch) (*EthTxReceipt, error)
-	EthGetCode(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
-	EthGetStorageAt(ctx context.Context, address ethtypes.EthAddress, position ethtypes.EthBytes, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
-	EthGetBalance(ctx context.Context, address ethtypes.EthAddress, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBigInt, error)
-	EthChainId(ctx context.Context) (ethtypes.EthUint64, error)
-	EthSyncing(ctx context.Context) (ethtypes.EthSyncingResult, error)
-	NetVersion(ctx context.Context) (string, error)
-	NetListening(ctx context.Context) (bool, error)
-	EthProtocolVersion(ctx context.Context) (ethtypes.EthUint64, error)
-	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
-	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
-	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
-	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
-	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)
-	EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
-	EthGetFilterLogs(ctx context.Context, id ethtypes.EthFilterID) (*ethtypes.EthFilterResult, error)
-	EthNewFilter(ctx context.Context, filter *ethtypes.EthFilterSpec) (ethtypes.EthFilterID, error)
-	EthNewBlockFilter(ctx context.Context) (ethtypes.EthFilterID, error)
-	EthNewPendingTransactionFilter(ctx context.Context) (ethtypes.EthFilterID, error)
-	EthUninstallFilter(ctx context.Context, id ethtypes.EthFilterID) (bool, error)
-	EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error)
-	EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error)
-	Web3ClientVersion(ctx context.Context) (string, error)
-	EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error)
-	EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error)
 }

api/api_lp.go

@@ -1,10 +0,0 @@
-package api
-
-import "context"
-
-type LotusProvider interface {
-	Version(context.Context) (Version, error) //perm:admin
-
-	// Trigger shutdown
-	Shutdown(context.Context) error //perm:admin
-}

api/api_net.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"time"
 
-	"github.com/libp2p/go-libp2p/core/metrics"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
-	"github.com/libp2p/go-libp2p/core/network"
+	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p/core/protocol"
+	"github.com/libp2p/go-libp2p-core/protocol"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -73,5 +73,5 @@ type CommonNet interface {
 
 type NatInfo struct {
 	Reachability network.Reachability
-	PublicAddrs  []string
+	PublicAddr   string
 }

api/api_storage.go

@@ -5,28 +5,28 @@ import (
 	"context"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
+	"github.com/filecoin-project/go-state-types/abi"
+	abinetwork "github.com/filecoin-project/go-state-types/network"
+	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
+	"github.com/filecoin-project/specs-storage/storage"
+
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/builtin/v9/market"
-	abinetwork "github.com/filecoin-project/go-state-types/network"
 
-	builtinactors "github.com/filecoin-project/lotus/chain/actors/builtin"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/storage/pipeline/sealiface"
+	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
-	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -50,22 +50,10 @@ type StorageMiner interface {
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
-	// WithdrawBalance allows to withdraw balance from miner actor to owner address
-	// Specify amount as "0" to withdraw full balance. This method returns a message CID
-	// and does not wait for message execution
-	ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin
-
-	// BeneficiaryWithdrawBalance allows the beneficiary of a miner to withdraw balance from miner actor
-	// Specify amount as "0" to withdraw full balance. This method returns a message CID
-	// and does not wait for message execution
-	BeneficiaryWithdrawBalance(context.Context, abi.TokenAmount) (cid.Cid, error) //perm:admin
-
 	MiningBase(context.Context) (*types.TipSet, error) //perm:read
 
 	ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
 
-	ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (abi.PieceInfo, error) //perm:admin
-
 	// Temp api for testing
 	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
 
@@ -75,9 +63,9 @@ type StorageMiner interface {
 	// Add piece to an open sector. If no sectors with enough space are open,
 	// either a new sector will be created, or this call will block until more
 	// sectors can be created.
-	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storiface.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
+	SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d PieceDealInfo) (SectorOffset, error) //perm:admin
 
-	SectorsUnsealPiece(ctx context.Context, sector storiface.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
+	SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin
 
 	// List all staged sectors
 	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
@@ -129,96 +117,53 @@ type StorageMiner interface {
 	SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
 	// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
 	SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
-	// SectorUnseal unseals the provided sector
-	SectorUnseal(ctx context.Context, number abi.SectorNumber) error //perm:admin
-
-	// SectorNumAssignerMeta returns sector number assigner metadata - reserved/allocated
-	SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
-	// SectorNumReservations returns a list of sector number reservations
-	SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
-	// SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
-	// numbers or name. Set force to true to override safety checks.
-	// Valid characters for name: a-z, A-Z, 0-9, _, -
-	SectorNumReserve(ctx context.Context, name string, sectors bitfield.BitField, force bool) error //perm:admin
-	// SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
-	// by default lotus will allocate lowest-available sector numbers to the reservation.
-	// For restrictions on `name` see SectorNumReserve
-	SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) //perm:admin
-	// SectorNumFree drops a sector reservation
-	SectorNumFree(ctx context.Context, name string) error //perm:admin
-
-	SectorReceive(ctx context.Context, meta RemoteSectorMeta) error //perm:admin
 
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
 	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
 	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
 
-	// storiface.WorkerReturn
+	//storiface.WorkerReturn
-	ReturnDataCid(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                      //perm:admin retry:true
-	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                                        //perm:admin retry:true
+	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storiface.PreCommit1Out, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
-	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storiface.SectorCids, err *storiface.CallError) error                       //perm:admin retry:true
+	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
-	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storiface.Commit1Out, err *storiface.CallError) error                             //perm:admin retry:true
+	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error                                //perm:admin retry:true
-	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storiface.Proof, err *storiface.CallError) error                                //perm:admin retry:true
+	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                  //perm:admin retry:true
-	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                    //perm:admin retry:true
+	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
-	ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storiface.ReplicaUpdateOut, err *storiface.CallError) error                     //perm:admin retry:true
+	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
-	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storiface.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
+	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
-	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storiface.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
+	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                         //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
-	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                             //perm:admin retry:true
+	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
-	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                   //perm:admin retry:true
+	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
-	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                       //perm:admin retry:true
+	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                              //perm:admin retry:true
-	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                                                //perm:admin retry:true
+	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                           //perm:admin retry:true
-	ReturnDownloadSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                    //perm:admin retry:true
-	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                             //perm:admin retry:true
 
 	// SealingSchedDiag dumps internal sealing scheduler state
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
-	// SealingSchedRemove removes a request from sealing pipeline
-	SealingRemoveRequest(ctx context.Context, schedId uuid.UUID) error //perm:admin
 
-	// paths.SectorIndex
+	// SectorIndex
-	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                         //perm:admin
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageDetach(ctx context.Context, id storiface.ID, url string) error                                                              //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
-	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                          //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
-	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                   //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
-	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                  //perm:admin
-	// StorageFindSector returns list of paths where the specified sector files exist.
-	//
-	// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
-	// - Paths which have sector files locally (don't require fetching) will be listed first.
-	// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
-	// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
-	//   file types are specified, each type will be considered individually, and a union of all paths
-	//   which can accommodate each file type will be returned.
 	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
-	// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
-	// Paths with more weight and more % of free space are preferred.
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
-	// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                   //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                        //perm:admin
-	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                  //perm:admin
-	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                          //perm:admin
 
 	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin
 
-	StorageAuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
+	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-
+	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
-	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
+	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
-	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
-	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
-
-	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error //perm:write
-	MarketListDeals(ctx context.Context) ([]*MarketDeal, error)                   //perm:read
-
-	// MarketListRetrievalDeals is deprecated, returns empty list
-	MarketListRetrievalDeals(ctx context.Context) ([]struct{}, error)                                                                                                                    //perm:read
 	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                    //perm:read
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
 	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
@@ -279,9 +224,6 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
-	// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
-	DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
-
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
 	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
@@ -297,7 +239,7 @@ type StorageMiner interface {
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
-	DealsList(ctx context.Context) ([]*MarketDeal, error)                        //perm:admin
+	DealsList(ctx context.Context) ([]MarketDeal, error)                         //perm:admin
 	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)               //perm:admin
 	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error              //perm:admin
 	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)             //perm:admin
@@ -313,6 +255,8 @@ type StorageMiner interface {
 	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
 	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
 
+	StorageAddLocal(ctx context.Context, path string) error //perm:admin
+
 	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
 	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
 	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
@@ -324,14 +268,9 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
-	ComputeProof(ctx context.Context, ssi []builtinactors.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtinactors.PoStProof, error) //perm:read
+	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
-
-	// RecoverFault can be used to declare recoveries manually. It sends messages
-	// to the miner actor with details of recovered sectors and returns the CID of messages. It honors the
-	// maxPartitionsPerRecoveryMessage from the config
-	RecoverFault(ctx context.Context, sectors []abi.SectorNumber) ([]cid.Cid, error) //perm:admin
 }
 
 var _ storiface.WorkerReturn = *new(StorageMiner)
@@ -421,10 +360,6 @@ func (st *SealSeed) Equals(ost *SealSeed) bool {
 
 type SectorState string
 
-func (s *SectorState) String() string {
-	return string(*s)
-}
-
 type AddrUse int
 
 const (
@@ -461,15 +396,10 @@ type SectorOffset struct {
 
 // DealInfo is a tuple of deal identity and its schedule
 type PieceDealInfo struct {
-	// "Old" builtin-market deal info
 	PublishCid   *cid.Cid
 	DealID       abi.DealID
 	DealProposal *market.DealProposal
-
-	// Common deal info
 	DealSchedule DealSchedule
-
-	// Best-effort deal asks
 	KeepUnsealed bool
 }
 
@@ -511,119 +441,3 @@ type DagstoreInitializeAllEvent struct {
 	Total   int
 	Current int
 }
-
-type NumAssignerMeta struct {
-	Reserved  bitfield.BitField
-	Allocated bitfield.BitField
-
-	// ChainAllocated+Reserved+Allocated
-	InUse bitfield.BitField
-
-	Next abi.SectorNumber
-}
-
-type RemoteSectorMeta struct {
-	////////
-	// BASIC SECTOR INFORMATION
-
-	// State specifies the first state the sector will enter after being imported
-	// Must be one of the following states:
-	// * Packing
-	// * GetTicket
-	// * PreCommitting
-	// * SubmitCommit
-	// * Proving/Available
-	State SectorState
-
-	Sector abi.SectorID
-	Type   abi.RegisteredSealProof
-
-	////////
-	// SEALING METADATA
-	// (allows lotus to continue the sealing process)
-
-	// Required in Packing and later
-	Pieces []SectorPiece // todo better type?
-
-	// Required in PreCommitting and later
-	TicketValue   abi.SealRandomness
-	TicketEpoch   abi.ChainEpoch
-	PreCommit1Out storiface.PreCommit1Out // todo specify better
-
-	CommD *cid.Cid
-	CommR *cid.Cid // SectorKey
-
-	// Required in SubmitCommit and later
-	PreCommitInfo    *miner.SectorPreCommitInfo
-	PreCommitDeposit *big.Int
-	PreCommitMessage *cid.Cid
-	PreCommitTipSet  types.TipSetKey
-
-	SeedValue abi.InteractiveSealRandomness
-	SeedEpoch abi.ChainEpoch
-
-	CommitProof []byte
-
-	// Required in Proving/Available
-	CommitMessage *cid.Cid
-
-	// Optional sector metadata to import
-	Log []SectorLog
-
-	////////
-	// SECTOR DATA SOURCE
-
-	// Sector urls - lotus will use those for fetching files into local storage
-
-	// Required in all states
-	DataUnsealed *storiface.SectorLocation
-
-	// Required in PreCommitting and later
-	DataSealed *storiface.SectorLocation
-	DataCache  *storiface.SectorLocation
-
-	////////
-	// SEALING SERVICE HOOKS
-
-	// URL
-	// RemoteCommit1Endpoint is an URL of POST endpoint which lotus will call requesting Commit1 (seal_commit_phase1)
-	// request body will be json-serialized RemoteCommit1Params struct
-	RemoteCommit1Endpoint string
-
-	// RemoteCommit2Endpoint is an URL of POST endpoint which lotus will call requesting Commit2 (seal_commit_phase2)
-	// request body will be json-serialized RemoteCommit2Params struct
-	RemoteCommit2Endpoint string
-
-	// RemoteSealingDoneEndpoint is called after the sector exists the sealing pipeline
-	// request body will be json-serialized RemoteSealingDoneParams struct
-	RemoteSealingDoneEndpoint string
-}
-
-type RemoteCommit1Params struct {
-	Ticket, Seed []byte
-
-	Unsealed cid.Cid
-	Sealed   cid.Cid
-
-	ProofType abi.RegisteredSealProof
-}
-
-type RemoteCommit2Params struct {
-	Sector    abi.SectorID
-	ProofType abi.RegisteredSealProof
-
-	// todo spec better
-	Commit1Out storiface.Commit1Out
-}
-
-type RemoteSealingDoneParams struct {
-	// Successful is true if the sector has entered state considered as "successfully sealed"
-	Successful bool
-
-	// State is the state the sector has entered
-	// For example "Proving" / "Removing"
-	State string
-
-	// Optional commit message CID
-	CommitMessage *cid.Cid
-}

api/api_test.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package api
 
 import (
@@ -12,9 +12,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-jsonrpc"
 )
 
 func goCmd() string {
@@ -127,18 +124,3 @@ func TestPermTags(t *testing.T) {
 	_ = PermissionedStorMinerAPI(&StorageMinerStruct{})
 	_ = PermissionedWorkerAPI(&WorkerStruct{})
 }
-
-func TestRetryErrorIsInTrue(t *testing.T) {
-	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
-	require.True(t, ErrorIsIn(&jsonrpc.RPCConnectionError{}, errorsToRetry))
-}
-
-func TestRetryErrorIsInFalse(t *testing.T) {
-	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
-	require.False(t, ErrorIsIn(xerrors.Errorf("random error"), errorsToRetry))
-}
-
-func TestRetryWrappedErrorIsInTrue(t *testing.T) {
-	errorsToRetry := []error{&jsonrpc.RPCConnectionError{}}
-	require.True(t, ErrorIsIn(xerrors.Errorf("wrapped: %w", &jsonrpc.RPCConnectionError{}), errorsToRetry))
-}

api/api_worker.go

@@ -7,10 +7,11 @@ import (
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/proof"
+	"github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
 
-	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/specs-storage/storage"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -33,23 +34,21 @@ type Worker interface {
 	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
 
 	// storiface.WorkerCalls
-	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                                                                                        //perm:admin
+	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
-	AddPiece(ctx context.Context, sector storiface.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storiface.Data) (storiface.CallID, error)                    //perm:admin
+	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
-	SealPreCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                             //perm:admin
+	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
-	SealPreCommit2(ctx context.Context, sector storiface.SectorRef, pc1o storiface.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
+	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
-	SealCommit1(ctx context.Context, sector storiface.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storiface.SectorCids) (storiface.CallID, error) //perm:admin
+	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
-	SealCommit2(ctx context.Context, sector storiface.SectorRef, c1o storiface.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
+	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	FinalizeSector(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error)                                                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
-	FinalizeReplicaUpdate(ctx context.Context, sector storiface.SectorRef) (storiface.CallID, error)                                                                                                         //perm:admin
+	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
-	ReplicaUpdate(ctx context.Context, sector storiface.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                         //perm:admin
+	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
-	ProveReplicaUpdate1(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                                //perm:admin
+	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
-	ProveReplicaUpdate2(ctx context.Context, sector storiface.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storiface.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
+	GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                    //perm:admin
-	GenerateSectorKeyFromData(ctx context.Context, sector storiface.SectorRef, commD cid.Cid) (storiface.CallID, error)                                                                                      //perm:admin
+	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
-	ReleaseUnsealed(ctx context.Context, sector storiface.SectorRef, keepUnsealed []storiface.Range) (storiface.CallID, error)                                                                               //perm:admin
+	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
-	MoveStorage(ctx context.Context, sector storiface.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                   //perm:admin
+	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
-	UnsealPiece(context.Context, storiface.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                             //perm:admin
+	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
-	Fetch(context.Context, storiface.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                               //perm:admin
-	DownloadSectorData(ctx context.Context, sector storiface.SectorRef, finalized bool, src map[storiface.SectorFileType]storiface.SectorLocation) (storiface.CallID, error)                                 //perm:admin
 
 	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)                           //perm:admin
 	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []storiface.PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (storiface.WindowPoStResult, error) //perm:admin
@@ -60,11 +59,7 @@ type Worker interface {
 	// Storage / Other
 	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)                   //perm:admin
+	StorageAddLocal(ctx context.Context, path string) error //perm:admin
-	StorageAddLocal(ctx context.Context, path string) error                              //perm:admin
-	StorageDetachLocal(ctx context.Context, path string) error                           //perm:admin
-	StorageDetachAll(ctx context.Context) error                                          //perm:admin
-	StorageRedeclareLocal(ctx context.Context, id *storiface.ID, dropMissing bool) error //perm:admin
 
 	// SetEnabled marks the worker as enabled/disabled. Not that this setting
 	// may take a few seconds to propagate to task scheduler
@@ -81,10 +76,6 @@ type Worker interface {
 
 	// Like ProcessSession, but returns an error when worker is disabled
 	Session(context.Context) (uuid.UUID, error) //perm:admin
-
-	// Trigger shutdown
-	Shutdown(context.Context) error //perm:admin
-
 }
 
 var _ storiface.WorkerCalls = *new(Worker)

api/client/client.go

@@ -15,21 +15,11 @@ import (
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
-// NewProviderRpc creates a new http jsonrpc client.
-func NewProviderRpc(ctx context.Context, addr string, requestHeader http.Header) (api.LotusProvider, jsonrpc.ClientCloser, error) {
-	var res v1api.LotusProviderStruct
-
-	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
-
-	return &res, closer, err
-}
-
 // NewCommonRPCV0 creates a new http jsonrpc client.
 func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
 	var res v0api.CommonNetStruct
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
+		api.GetInternalStructs(&res), requestHeader)
 
 	return &res, closer, err
 }
@@ -39,16 +29,16 @@ func NewFullNodeRPCV0(ctx context.Context, addr string, requestHeader http.Heade
 	var res v0api.FullNodeStruct
 
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
+		api.GetInternalStructs(&res), requestHeader)
 
 	return &res, closer, err
 }
 
 // NewFullNodeRPCV1 creates a new http jsonrpc client.
-func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.FullNode, jsonrpc.ClientCloser, error) {
+func NewFullNodeRPCV1(ctx context.Context, addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
 	var res v1api.FullNodeStruct
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
-		api.GetInternalStructs(&res), requestHeader, append([]jsonrpc.Option{jsonrpc.WithErrors(api.RPCErrors)}, opts...)...)
+		api.GetInternalStructs(&res), requestHeader)
 
 	return &res, closer, err
 }
@@ -82,7 +72,6 @@ func NewStorageMinerRPCV0(ctx context.Context, addr string, requestHeader http.H
 		api.GetInternalStructs(&res), requestHeader,
 		append([]jsonrpc.Option{
 			rpcenc.ReaderParamEncoder(pushUrl),
-			jsonrpc.WithErrors(api.RPCErrors),
 		}, opts...)...)
 
 	return &res, closer, err
@@ -101,7 +90,6 @@ func NewWorkerRPCV0(ctx context.Context, addr string, requestHeader http.Header)
 		rpcenc.ReaderParamEncoder(pushUrl),
 		jsonrpc.WithNoReconnect(),
 		jsonrpc.WithTimeout(30*time.Second),
-		jsonrpc.WithErrors(api.RPCErrors),
 	)
 
 	return &res, closer, err
@@ -113,7 +101,7 @@ func NewGatewayRPCV1(ctx context.Context, addr string, requestHeader http.Header
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		api.GetInternalStructs(&res),
 		requestHeader,
-		append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
+		opts...,
 	)
 
 	return &res, closer, err
@@ -125,7 +113,7 @@ func NewGatewayRPCV0(ctx context.Context, addr string, requestHeader http.Header
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		api.GetInternalStructs(&res),
 		requestHeader,
-		append(opts, jsonrpc.WithErrors(api.RPCErrors))...,
+		opts...,
 	)
 
 	return &res, closer, err
@@ -136,7 +124,6 @@ func NewWalletRPCV0(ctx context.Context, addr string, requestHeader http.Header)
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		api.GetInternalStructs(&res),
 		requestHeader,
-		jsonrpc.WithErrors(api.RPCErrors),
 	)
 
 	return &res, closer, err

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -8,6 +8,7 @@ import (
 	"os"
 
 	"github.com/filecoin-project/lotus/api/docgen"
+
 	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
 )
 

api/docgen-openrpc/openrpc.go

@@ -8,11 +8,10 @@ import (
 
 	"github.com/alecthomas/jsonschema"
 	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
-	"github.com/ipfs/go-cid"
-	meta_schema "github.com/open-rpc/meta-schema"
-
 	"github.com/filecoin-project/lotus/api/docgen"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/ipfs/go-cid"
+	meta_schema "github.com/open-rpc/meta-schema"
 )
 
 // schemaDictEntry represents a type association passed to the jsonschema reflector.

api/docgen/docgen.go

@@ -6,33 +6,31 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"net/http"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"time"
 	"unicode"
 
-	"github.com/google/uuid"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-graphsync"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/libp2p/go-libp2p/core/metrics"
-	"github.com/libp2p/go-libp2p/core/network"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/protocol"
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
+	"github.com/google/uuid"
-	"github.com/filecoin-project/go-fil-markets/filestore"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
+	"github.com/libp2p/go-libp2p-core/metrics"
+	"github.com/libp2p/go-libp2p-core/network"
+	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p-core/protocol"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/multiformats/go-multiaddr"
+
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	filestore "github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
+
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/exitcode"
 
@@ -41,12 +39,11 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/types/ethtypes"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
-	sealing "github.com/filecoin-project/lotus/storage/pipeline"
-	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var ExampleValues = map[reflect.Type]interface{}{
@@ -69,7 +66,6 @@ func init() {
 	}
 
 	ExampleValues[reflect.TypeOf(c)] = c
-	ExampleValues[reflect.TypeOf(&c)] = &c
 
 	c2, err := cid.Decode("bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve")
 	if err != nil {
@@ -86,7 +82,6 @@ func init() {
 	}
 
 	ExampleValues[reflect.TypeOf(addr)] = addr
-	ExampleValues[reflect.TypeOf(&addr)] = &addr
 
 	pid, err := peer.Decode("12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf")
 	if err != nil {
@@ -100,9 +95,6 @@ func init() {
 	apiSelExample := api.Selector("Links/21/Hash/Links/42/Hash")
 	clientEvent := retrievalmarket.ClientEventDealAccepted
 
-	block := blocks.Block(&blocks.BasicBlock{})
-	ExampleValues[reflect.TypeOf(&block).Elem()] = block
-
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
 	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
@@ -141,20 +133,12 @@ func init() {
 	addExample(&textSelExample)
 	addExample(&apiSelExample)
 	addExample(network.ReachabilityPublic)
-	addExample(build.TestNetworkVersion)
+	addExample(build.NewestNetworkVersion)
-	allocationId := verifreg.AllocationId(0)
-	addExample(allocationId)
-	addExample(&allocationId)
-	addExample(map[verifreg.AllocationId]verifreg.Allocation{})
-	claimId := verifreg.ClaimId(0)
-	addExample(claimId)
-	addExample(&claimId)
-	addExample(map[verifreg.ClaimId]verifreg.Claim{})
 	addExample(map[string]int{"name": 42})
 	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
-		Msg:    ExampleValue("init", reflect.TypeOf(types.MessageTrace{}), nil).(types.MessageTrace),
+		Msg:    ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: ExampleValue("init", reflect.TypeOf(types.ReturnTrace{}), nil).(types.ReturnTrace),
+		MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
 	})
 	addExample(map[string]types.Actor{
 		"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
@@ -162,10 +146,6 @@ func init() {
 	addExample(map[string]api.MarketDeal{
 		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
 	})
-	addExample(map[string]*api.MarketDeal{
-		"t026363": ExampleValue("init", reflect.TypeOf(&api.MarketDeal{}), nil).(*api.MarketDeal),
-	})
-
 	addExample(map[string]api.MarketBalance{
 		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
 	})
@@ -208,7 +188,6 @@ func init() {
 	addExample(&si)
 	addExample(retrievalmarket.DealID(5))
 	addExample(abi.ActorID(1000))
-	addExample(map[string]cid.Cid{})
 	addExample(map[string][]api.SealedRef{
 		"98000": {
 			api.SealedRef{
@@ -285,8 +264,6 @@ func init() {
 			Read:   [storiface.FileTypes]uint{2, 3, 0},
 		},
 	})
-	storifaceid := storiface.ID("1399aa04-2625-44b1-bad4-bd07b59b22c4")
-	addExample(&storifaceid)
 
 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -301,8 +278,7 @@ func init() {
 			"title":   "Lotus RPC API",
 			"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
 		},
-		"methods": []interface{}{},
+		"methods": []interface{}{}},
-	},
 	)
 
 	addExample(api.CheckStatusCode(0))
@@ -339,8 +315,7 @@ func init() {
 			NumConnsInbound:    3,
 			NumConnsOutbound:   4,
 			NumFD:              5,
-		},
+		}})
-	})
 	addExample(api.NetLimit{
 		Memory:          123,
 		StreamsInbound:  1,
@@ -352,59 +327,10 @@ func init() {
 		FD:              5,
 	})
 
-	addExample(map[string]bitfield.BitField{
-		"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
-	})
-
-	addExample(http.Header{
-		"Authorization": []string{"Bearer ey.."},
-	})
-
-	addExample(map[storiface.SectorFileType]storiface.SectorLocation{
-		storiface.FTSealed: {
-			Local:   false,
-			URL:     "https://example.com/sealingservice/sectors/s-f0123-12345",
-			Headers: nil,
-		},
-	})
-
-	ethint := ethtypes.EthUint64(5)
-	addExample(ethint)
-	addExample(&ethint)
-
-	ethaddr, _ := ethtypes.ParseEthAddress("0x5CbEeCF99d3fDB3f25E309Cc264f240bb0664031")
-	addExample(ethaddr)
-	addExample(&ethaddr)
-
-	ethhash, _ := ethtypes.EthHashFromCid(c)
-	addExample(ethhash)
-	addExample(&ethhash)
-
-	ethFeeHistoryReward := [][]ethtypes.EthBigInt{}
-	addExample(&ethFeeHistoryReward)
-
-	addExample(&uuid.UUID{})
-
-	filterid := ethtypes.EthFilterID(ethhash)
-	addExample(filterid)
-	addExample(&filterid)
-
-	subid := ethtypes.EthSubscriptionID(ethhash)
-	addExample(subid)
-	addExample(&subid)
-
-	pstring := func(s string) *string { return &s }
-	addExample(&ethtypes.EthFilterSpec{
-		FromBlock: pstring("2301220"),
-		Address:   []ethtypes.EthAddress{ethaddr},
-	})
-
-	percent := types.Percent(123)
-	addExample(percent)
-	addExample(&percent)
 }
 
 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
+
 	switch pkg {
 	case "api": // latest
 		switch name {
@@ -424,14 +350,6 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
 			i = &api.WorkerStruct{}
 			t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
 			permStruct = append(permStruct, reflect.TypeOf(api.WorkerStruct{}.Internal))
-		case "Gateway":
-			i = &api.GatewayStruct{}
-			t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
-		case "Provider":
-			i = &api.LotusProviderStruct{}
-			t = reflect.TypeOf(new(struct{ api.LotusProvider })).Elem()
-			permStruct = append(permStruct, reflect.TypeOf(api.LotusProviderStruct{}.Internal))
 		default:
 			panic("unknown type")
 		}
@@ -478,11 +396,8 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	case reflect.Ptr:
 		if t.Elem().Kind() == reflect.Struct {
 			es := exampleStruct(method, t.Elem(), t)
-			ExampleValues[t] = es
+			//ExampleValues[t] = es
 			return es
-		} else if t.Elem().Kind() == reflect.String {
-			str := "string value"
-			return &str
 		}
 	case reflect.Interface:
 		return struct{}{}
@@ -498,8 +413,7 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
 		if f.Type == parent {
 			continue
 		}
-
+		if strings.Title(f.Name) == f.Name {
-		if f.IsExported() {
 			ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
 		}
 	}

api/eth_aliases.go

@@ -1,50 +0,0 @@
-package api
-
-import apitypes "github.com/filecoin-project/lotus/api/types"
-
-func CreateEthRPCAliases(as apitypes.Aliaser) {
-	// TODO: maybe use reflect to automatically register all the eth aliases
-	as.AliasMethod("eth_accounts", "Filecoin.EthAccounts")
-	as.AliasMethod("eth_blockNumber", "Filecoin.EthBlockNumber")
-	as.AliasMethod("eth_getBlockTransactionCountByNumber", "Filecoin.EthGetBlockTransactionCountByNumber")
-	as.AliasMethod("eth_getBlockTransactionCountByHash", "Filecoin.EthGetBlockTransactionCountByHash")
-
-	as.AliasMethod("eth_getBlockByHash", "Filecoin.EthGetBlockByHash")
-	as.AliasMethod("eth_getBlockByNumber", "Filecoin.EthGetBlockByNumber")
-	as.AliasMethod("eth_getTransactionByHash", "Filecoin.EthGetTransactionByHash")
-	as.AliasMethod("eth_getTransactionCount", "Filecoin.EthGetTransactionCount")
-	as.AliasMethod("eth_getTransactionReceipt", "Filecoin.EthGetTransactionReceipt")
-	as.AliasMethod("eth_getTransactionByBlockHashAndIndex", "Filecoin.EthGetTransactionByBlockHashAndIndex")
-	as.AliasMethod("eth_getTransactionByBlockNumberAndIndex", "Filecoin.EthGetTransactionByBlockNumberAndIndex")
-
-	as.AliasMethod("eth_getCode", "Filecoin.EthGetCode")
-	as.AliasMethod("eth_getStorageAt", "Filecoin.EthGetStorageAt")
-	as.AliasMethod("eth_getBalance", "Filecoin.EthGetBalance")
-	as.AliasMethod("eth_chainId", "Filecoin.EthChainId")
-	as.AliasMethod("eth_syncing", "Filecoin.EthSyncing")
-	as.AliasMethod("eth_feeHistory", "Filecoin.EthFeeHistory")
-	as.AliasMethod("eth_protocolVersion", "Filecoin.EthProtocolVersion")
-	as.AliasMethod("eth_maxPriorityFeePerGas", "Filecoin.EthMaxPriorityFeePerGas")
-	as.AliasMethod("eth_gasPrice", "Filecoin.EthGasPrice")
-	as.AliasMethod("eth_sendRawTransaction", "Filecoin.EthSendRawTransaction")
-	as.AliasMethod("eth_estimateGas", "Filecoin.EthEstimateGas")
-	as.AliasMethod("eth_call", "Filecoin.EthCall")
-
-	as.AliasMethod("eth_getLogs", "Filecoin.EthGetLogs")
-	as.AliasMethod("eth_getFilterChanges", "Filecoin.EthGetFilterChanges")
-	as.AliasMethod("eth_getFilterLogs", "Filecoin.EthGetFilterLogs")
-	as.AliasMethod("eth_newFilter", "Filecoin.EthNewFilter")
-	as.AliasMethod("eth_newBlockFilter", "Filecoin.EthNewBlockFilter")
-	as.AliasMethod("eth_newPendingTransactionFilter", "Filecoin.EthNewPendingTransactionFilter")
-	as.AliasMethod("eth_uninstallFilter", "Filecoin.EthUninstallFilter")
-	as.AliasMethod("eth_subscribe", "Filecoin.EthSubscribe")
-	as.AliasMethod("eth_unsubscribe", "Filecoin.EthUnsubscribe")
-
-	as.AliasMethod("trace_block", "Filecoin.EthTraceBlock")
-	as.AliasMethod("trace_replayBlockTransactions", "Filecoin.EthTraceReplayBlockTransactions")
-
-	as.AliasMethod("net_version", "Filecoin.NetVersion")
-	as.AliasMethod("net_listening", "Filecoin.NetListening")
-
-	as.AliasMethod("web3_clientVersion", "Filecoin.Web3ClientVersion")
-}

api/permissioned.go

@@ -41,12 +41,6 @@ func PermissionedWorkerAPI(a Worker) Worker {
 	return &out
 }
 
-func PermissionedAPI[T, P any](a T) *P {
-	var out P
-	permissionedProxies(a, &out)
-	return &out
-}
-
 func PermissionedWalletAPI(a Wallet) Wallet {
 	var out WalletStruct
 	permissionedProxies(a, &out)

api/proxy_util_test.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package api
 
 import (

api/types.go

@@ -5,26 +5,22 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/google/uuid"
+	"github.com/libp2p/go-libp2p-core/network"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-graphsync"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/codec/dagjson"
-	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/libp2p/go-libp2p/core/network"
-	"github.com/libp2p/go-libp2p/core/peer"
-	ma "github.com/multiformats/go-multiaddr"
 
-	"github.com/filecoin-project/go-address"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/abi"
-
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
+
+	"github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	ma "github.com/multiformats/go-multiaddr"
 )
 
+// TODO: check if this exists anywhere else
+
 type MultiaddrSlice []ma.Multiaddr
 
 func (m *MultiaddrSlice) UnmarshalJSON(raw []byte) (err error) {
@@ -56,17 +52,8 @@ type PubsubScore struct {
 	Score *pubsub.PeerScoreSnapshot
 }
 
-// MessageSendSpec contains optional fields which modify message sending behavior
 type MessageSendSpec struct {
-	// MaxFee specifies a cap on network fees related to this message
 	MaxFee abi.TokenAmount
-
-	// MsgUuid specifies a unique message identifier which can be used on node (or node cluster)
-	// level to prevent double-sends of messages even when nonce generation is not handled by sender
-	MsgUuid uuid.UUID
-
-	// MaximizeFeeCap makes message FeeCap be based entirely on MaxFee
-	MaximizeFeeCap bool
 }
 
 // GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
@@ -115,12 +102,16 @@ func NewDataTransferChannel(hostID peer.ID, channelState datatransfer.ChannelSta
 		IsSender:   channelState.Sender() == hostID,
 		Message:    channelState.Message(),
 	}
-	voucher := channelState.Voucher()
+	stringer, ok := channelState.Voucher().(fmt.Stringer)
-	voucherJSON, err := ipld.Encode(voucher.Voucher, dagjson.Encode)
+	if ok {
-	if err != nil {
+		channel.Voucher = stringer.String()
-		channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
 	} else {
-		channel.Voucher = string(voucherJSON)
+		voucherJSON, err := json.Marshal(channelState.Voucher())
+		if err != nil {
+			channel.Voucher = fmt.Errorf("Voucher Serialization: %w", err).Error()
+		} else {
+			channel.Voucher = string(voucherJSON)
+		}
 	}
 	if channel.IsSender {
 		channel.IsInitiator = !channelState.IsPull()
@@ -143,7 +134,13 @@ type NetStat struct {
 }
 
 type NetLimit struct {
+	Dynamic bool `json:",omitempty"`
+	// set if Dynamic is false
 	Memory int64 `json:",omitempty"`
+	// set if Dynamic is true
+	MemoryFraction float64 `json:",omitempty"`
+	MinMemory      int64   `json:",omitempty"`
+	MaxMemory      int64   `json:",omitempty"`
 
 	Streams, StreamsInbound, StreamsOutbound int
 	Conns, ConnsInbound, ConnsOutbound       int
@@ -255,10 +252,10 @@ type RestrievalRes struct {
 }
 
 // Selector specifies ipld selector string
-//   - if the string starts with '{', it's interpreted as json selector string
+// - if the string starts with '{', it's interpreted as json selector string
-//     see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
+//   see https://ipld.io/specs/selectors/ and https://ipld.io/specs/selectors/fixtures/selector-fixtures-1/
-//   - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
+// - otherwise the string is interpreted as ipld-selector-text-lite (simple ipld path)
-//     see https://github.com/ipld/go-ipld-selector-text-lite
+//   see https://github.com/ipld/go-ipld-selector-text-lite
 type Selector string
 
 type DagSpec struct {
@@ -289,68 +286,3 @@ type ExportRef struct {
 	FromLocalCAR string // if specified, get data from a local CARv2 file.
 	DealID       retrievalmarket.DealID
 }
-
-type MinerInfo struct {
-	Owner                      address.Address   // Must be an ID-address.
-	Worker                     address.Address   // Must be an ID-address.
-	NewWorker                  address.Address   // Must be an ID-address.
-	ControlAddresses           []address.Address // Must be an ID-addresses.
-	WorkerChangeEpoch          abi.ChainEpoch
-	PeerId                     *peer.ID
-	Multiaddrs                 []abi.Multiaddrs
-	WindowPoStProofType        abi.RegisteredPoStProof
-	SectorSize                 abi.SectorSize
-	WindowPoStPartitionSectors uint64
-	ConsensusFaultElapsed      abi.ChainEpoch
-	PendingOwnerAddress        *address.Address
-	Beneficiary                address.Address
-	BeneficiaryTerm            *miner.BeneficiaryTerm
-	PendingBeneficiaryTerm     *miner.PendingBeneficiaryChange
-}
-
-type NetworkParams struct {
-	NetworkName             dtypes.NetworkName
-	BlockDelaySecs          uint64
-	ConsensusMinerMinPower  abi.StoragePower
-	SupportedProofTypes     []abi.RegisteredSealProof
-	PreCommitChallengeDelay abi.ChainEpoch
-	ForkUpgradeParams       ForkUpgradeParams
-	Eip155ChainID           int
-}
-
-type ForkUpgradeParams struct {
-	UpgradeSmokeHeight       abi.ChainEpoch
-	UpgradeBreezeHeight      abi.ChainEpoch
-	UpgradeIgnitionHeight    abi.ChainEpoch
-	UpgradeLiftoffHeight     abi.ChainEpoch
-	UpgradeAssemblyHeight    abi.ChainEpoch
-	UpgradeRefuelHeight      abi.ChainEpoch
-	UpgradeTapeHeight        abi.ChainEpoch
-	UpgradeKumquatHeight     abi.ChainEpoch
-	BreezeGasTampingDuration abi.ChainEpoch
-	UpgradeCalicoHeight      abi.ChainEpoch
-	UpgradePersianHeight     abi.ChainEpoch
-	UpgradeOrangeHeight      abi.ChainEpoch
-	UpgradeClausHeight       abi.ChainEpoch
-	UpgradeTrustHeight       abi.ChainEpoch
-	UpgradeNorwegianHeight   abi.ChainEpoch
-	UpgradeTurboHeight       abi.ChainEpoch
-	UpgradeHyperdriveHeight  abi.ChainEpoch
-	UpgradeChocolateHeight   abi.ChainEpoch
-	UpgradeOhSnapHeight      abi.ChainEpoch
-	UpgradeSkyrHeight        abi.ChainEpoch
-	UpgradeSharkHeight       abi.ChainEpoch
-	UpgradeHyggeHeight       abi.ChainEpoch
-	UpgradeLightningHeight   abi.ChainEpoch
-	UpgradeThunderHeight     abi.ChainEpoch
-	UpgradeWatermelonHeight  abi.ChainEpoch
-}
-
-// ChainExportConfig holds configuration for chain ranged exports.
-type ChainExportConfig struct {
-	WriteBufferSize   int
-	NumWorkers        int
-	IncludeMessages   bool
-	IncludeReceipts   bool
-	IncludeStateRoots bool
-}

api/types/rpc.go

@@ -1,5 +0,0 @@
-package apitypes
-
-type Aliaser interface {
-	AliasMethod(alias, original string)
-}

api/v0api/full.go

@@ -3,26 +3,22 @@ package v0api
 import (
 	"context"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-	"github.com/libp2p/go-libp2p/core/peer"
-
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
-	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
-	abinetwork "github.com/filecoin-project/go-state-types/network"
+	"github.com/ipfs/go-cid"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -113,9 +109,6 @@ type FullNode interface {
 	// ChainDeleteObj deletes node referenced by the given CID
 	ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
 
-	// ChainPutObj puts and object into the blockstore
-	ChainPutObj(context.Context, blocks.Block) error
-
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
 	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
 
@@ -140,7 +133,7 @@ type FullNode interface {
 
 	// ChainGetPath returns a set of revert/apply operations needed to get from
 	// one tipset to another, for example:
-	// ```
+	//```
 	//        to
 	//         ^
 	// from   tAA
@@ -149,7 +142,7 @@ type FullNode interface {
 	//  ^---*--^
 	//      ^
 	//     tRR
-	// ```
+	//```
 	// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*api.HeadChange, error) //perm:read
 
@@ -366,12 +359,12 @@ type FullNode interface {
 	ClientCancelRetrievalDeal(ctx context.Context, dealid retrievalmarket.DealID) error //perm:write
 
 	// ClientUnimport removes references to the specified file from filestore
-	// ClientUnimport(path string)
+	//ClientUnimport(path string)
 
 	// ClientListImports lists imported files and their root CIDs
 	ClientListImports(ctx context.Context) ([]api.Import, error) //perm:write
 
-	// ClientListAsks() []Ask
+	//ClientListAsks() []Ask
 
 	// MethodGroup: State
 	// The State methods are used to query, inspect, and interact with chain state.
@@ -386,7 +379,7 @@ type FullNode interface {
 	StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
 	//
-	// If a tipset key is provided, and a replacing message is not found on chain,
+	// If a tipset key is provided, and a replacing message is found on chain,
 	// the method will return an error saying that the message wasn't found
 	//
 	// If no tipset key is provided, the appropriate tipset is looked up, and if
@@ -423,7 +416,7 @@ type FullNode interface {
 	// StateMinerPower returns the power of the indicated miner
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error) //perm:read
 	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) //perm:read
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
 	// StateMinerDeadlines returns all the proving deadlines for the given miner
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error) //perm:read
 	// StateMinerPartitions returns all partitions in the specified deadline
@@ -528,19 +521,9 @@ type FullNode interface {
 	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
 	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]api.MarketBalance, error) //perm:read
 	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]*api.MarketDeal, error) //perm:read
+	StateMarketDeals(context.Context, types.TipSetKey) (map[string]api.MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
 	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*api.MarketDeal, error) //perm:read
-	// StateGetAllocationForPendingDeal returns the allocation for a given deal ID of a pending deal.
-	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
-	// StateGetAllocation returns the allocation for a given address and allocation ID.
-	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error) //perm:read
-	// StateGetAllocations returns the all the allocations for a given client.
-	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) //perm:read
-	// StateGetClaim returns the claim for a given address and claim ID.
-	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error) //perm:read
-	// StateGetClaims returns the all the claims for a given provider.
-	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
 	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateAccountKey returns the public key address of the given ID address
@@ -601,7 +584,7 @@ type FullNode interface {
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
-	// StateVerifiedRegistryRootKey returns the address of the Verified Registry's root key
+	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
 	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
 	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
 	// can issue. It takes the deal size and verified status as parameters.
@@ -615,19 +598,12 @@ type FullNode interface {
 	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) //perm:read
 	// StateNetworkVersion returns the network version at the given tipset
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
-	// StateActorCodeCIDs returns the CIDs of all the builtin actors for the given network version
-	StateActorCodeCIDs(context.Context, abinetwork.Version) (map[string]cid.Cid, error) //perm:read
-	// StateActorManifestCID returns the CID of the builtin actors manifest for the given network version
-	StateActorManifestCID(context.Context, abinetwork.Version) (cid.Cid, error) //perm:read
 
 	// StateGetRandomnessFromTickets is used to sample the chain for randomness.
 	StateGetRandomnessFromTickets(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 	// StateGetRandomnessFromBeacon is used to sample the beacon for randomness.
 	StateGetRandomnessFromBeacon(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte, tsk types.TipSetKey) (abi.Randomness, error) //perm:read
 
-	// StateGetNetworkParams return current network params
-	StateGetNetworkParams(ctx context.Context) (*api.NetworkParams, error) //perm:read
-
 	// MethodGroup: Msig
 	// The Msig methods are used to interact with multisig wallets on the
 	// filecoin network
@@ -640,14 +616,14 @@ type FullNode interface {
 	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
 	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
 
-	// MsigGetPending returns pending transactions for the given multisig
+	//MsigGetPending returns pending transactions for the given multisig
-	// wallet. Once pending transactions are fully approved, they will no longer
+	//wallet. Once pending transactions are fully approved, they will no longer
-	// appear here.
+	//appear here.
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error) //perm:read
 
 	// MsigCreate creates a multisig wallet
 	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
-	// <initial balance>, <sender address of the create msg>, <gas price>
+	//<initial balance>, <sender address of the create msg>, <gas price>
 	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
 	// MsigPropose proposes a multisig message
 	// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,

api/v0api/gateway.go

@@ -3,19 +3,16 @@ package v0api
 import (
 	"context"
 
-	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
-	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -36,14 +33,7 @@ import (
 //  * Generate openrpc blobs
 
 type Gateway interface {
-	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
-	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
-	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*api.MiningBaseInfo, error)
-	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)
-	GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
-	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
 	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
@@ -52,30 +42,21 @@ type Gateway interface {
 	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
 	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*api.MsigTransaction, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
-	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
-	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
-	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
-	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
-	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
 	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
 	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
-	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (api.MinerInfo, error)
+	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
-	StateNetworkName(context.Context) (dtypes.NetworkName, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (abinetwork.Version, error)
 	StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)

api/v0api/permissioned.go

@@ -2,7 +2,6 @@ package v0api
 
 import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
-
 	"github.com/filecoin-project/lotus/api"
 )
 

api/v0api/v0mocks/mock_full.go

@@ -9,39 +9,35 @@ import (
 	reflect "reflect"
 	time "time"
 
-	gomock "github.com/golang/mock/gomock"
-	uuid "github.com/google/uuid"
-	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
-	metrics "github.com/libp2p/go-libp2p/core/metrics"
-	network0 "github.com/libp2p/go-libp2p/core/network"
-	peer "github.com/libp2p/go-libp2p/core/peer"
-	protocol "github.com/libp2p/go-libp2p/core/protocol"
-
 	address "github.com/filecoin-project/go-address"
 	bitfield "github.com/filecoin-project/go-bitfield"
-	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
 	retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
 	auth "github.com/filecoin-project/go-jsonrpc/auth"
 	abi "github.com/filecoin-project/go-state-types/abi"
 	big "github.com/filecoin-project/go-state-types/big"
-	paych "github.com/filecoin-project/go-state-types/builtin/v8/paych"
-	miner "github.com/filecoin-project/go-state-types/builtin/v9/miner"
-	verifreg "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	crypto "github.com/filecoin-project/go-state-types/crypto"
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
-
 	api "github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	v0api "github.com/filecoin-project/lotus/api/v0api"
-	miner0 "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	alerting "github.com/filecoin-project/lotus/journal/alerting"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 	dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	imports "github.com/filecoin-project/lotus/node/repo/imports"
+	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	gomock "github.com/golang/mock/gomock"
+	uuid "github.com/google/uuid"
+	cid "github.com/ipfs/go-cid"
+	metrics "github.com/libp2p/go-libp2p-core/metrics"
+	network0 "github.com/libp2p/go-libp2p-core/network"
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	protocol "github.com/libp2p/go-libp2p-core/protocol"
 )
 
 // MockFullNode is a mock of FullNode interface.
@@ -381,20 +377,6 @@ func (mr *MockFullNodeMockRecorder) ChainNotify(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainNotify", reflect.TypeOf((*MockFullNode)(nil).ChainNotify), arg0)
 }
 
-// ChainPutObj mocks base method.
-func (m *MockFullNode) ChainPutObj(arg0 context.Context, arg1 blocks.Block) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ChainPutObj", arg0, arg1)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// ChainPutObj indicates an expected call of ChainPutObj.
-func (mr *MockFullNodeMockRecorder) ChainPutObj(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainPutObj", reflect.TypeOf((*MockFullNode)(nil).ChainPutObj), arg0, arg1)
-}
-
 // ChainReadObj mocks base method.
 func (m *MockFullNode) ChainReadObj(arg0 context.Context, arg1 cid.Cid) ([]byte, error) {
 	m.ctrl.T.Helper()
@@ -2158,21 +2140,6 @@ func (mr *MockFullNodeMockRecorder) Shutdown(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockFullNode)(nil).Shutdown), arg0)
 }
 
-// StartTime mocks base method.
-func (m *MockFullNode) StartTime(arg0 context.Context) (time.Time, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StartTime", arg0)
-	ret0, _ := ret[0].(time.Time)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StartTime indicates an expected call of StartTime.
-func (mr *MockFullNodeMockRecorder) StartTime(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartTime", reflect.TypeOf((*MockFullNode)(nil).StartTime), arg0)
-}
-
 // StateAccountKey mocks base method.
 func (m *MockFullNode) StateAccountKey(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (address.Address, error) {
 	m.ctrl.T.Helper()
@@ -2188,36 +2155,6 @@ func (mr *MockFullNodeMockRecorder) StateAccountKey(arg0, arg1, arg2 interface{}
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateAccountKey", reflect.TypeOf((*MockFullNode)(nil).StateAccountKey), arg0, arg1, arg2)
 }
 
-// StateActorCodeCIDs mocks base method.
-func (m *MockFullNode) StateActorCodeCIDs(arg0 context.Context, arg1 network.Version) (map[string]cid.Cid, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateActorCodeCIDs", arg0, arg1)
-	ret0, _ := ret[0].(map[string]cid.Cid)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateActorCodeCIDs indicates an expected call of StateActorCodeCIDs.
-func (mr *MockFullNodeMockRecorder) StateActorCodeCIDs(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorCodeCIDs", reflect.TypeOf((*MockFullNode)(nil).StateActorCodeCIDs), arg0, arg1)
-}
-
-// StateActorManifestCID mocks base method.
-func (m *MockFullNode) StateActorManifestCID(arg0 context.Context, arg1 network.Version) (cid.Cid, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateActorManifestCID", arg0, arg1)
-	ret0, _ := ret[0].(cid.Cid)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateActorManifestCID indicates an expected call of StateActorManifestCID.
-func (mr *MockFullNodeMockRecorder) StateActorManifestCID(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateActorManifestCID", reflect.TypeOf((*MockFullNode)(nil).StateActorManifestCID), arg0, arg1)
-}
-
 // StateAllMinerFaults mocks base method.
 func (m *MockFullNode) StateAllMinerFaults(arg0 context.Context, arg1 abi.ChainEpoch, arg2 types.TipSetKey) ([]*api.Fault, error) {
 	m.ctrl.T.Helper()
@@ -2249,10 +2186,10 @@ func (mr *MockFullNodeMockRecorder) StateCall(arg0, arg1, arg2 interface{}) *gom
 }
 
 // StateChangedActors mocks base method.
-func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.ActorV5, error) {
+func (m *MockFullNode) StateChangedActors(arg0 context.Context, arg1, arg2 cid.Cid) (map[string]types.Actor, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateChangedActors", arg0, arg1, arg2)
-	ret0, _ := ret[0].(map[string]types.ActorV5)
+	ret0, _ := ret[0].(map[string]types.Actor)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2324,10 +2261,10 @@ func (mr *MockFullNodeMockRecorder) StateDecodeParams(arg0, arg1, arg2, arg3, ar
 }
 
 // StateGetActor mocks base method.
-func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.ActorV5, error) {
+func (m *MockFullNode) StateGetActor(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (*types.Actor, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateGetActor", arg0, arg1, arg2)
-	ret0, _ := ret[0].(*types.ActorV5)
+	ret0, _ := ret[0].(*types.Actor)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2338,96 +2275,6 @@ func (mr *MockFullNodeMockRecorder) StateGetActor(arg0, arg1, arg2 interface{})
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetActor", reflect.TypeOf((*MockFullNode)(nil).StateGetActor), arg0, arg1, arg2)
 }
 
-// StateGetAllocation mocks base method.
-func (m *MockFullNode) StateGetAllocation(arg0 context.Context, arg1 address.Address, arg2 verifreg.AllocationId, arg3 types.TipSetKey) (*verifreg.Allocation, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetAllocation", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*verifreg.Allocation)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetAllocation indicates an expected call of StateGetAllocation.
-func (mr *MockFullNodeMockRecorder) StateGetAllocation(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocation", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocation), arg0, arg1, arg2, arg3)
-}
-
-// StateGetAllocationForPendingDeal mocks base method.
-func (m *MockFullNode) StateGetAllocationForPendingDeal(arg0 context.Context, arg1 abi.DealID, arg2 types.TipSetKey) (*verifreg.Allocation, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetAllocationForPendingDeal", arg0, arg1, arg2)
-	ret0, _ := ret[0].(*verifreg.Allocation)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetAllocationForPendingDeal indicates an expected call of StateGetAllocationForPendingDeal.
-func (mr *MockFullNodeMockRecorder) StateGetAllocationForPendingDeal(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocationForPendingDeal", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocationForPendingDeal), arg0, arg1, arg2)
-}
-
-// StateGetAllocations mocks base method.
-func (m *MockFullNode) StateGetAllocations(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.AllocationId]verifreg.Allocation, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetAllocations", arg0, arg1, arg2)
-	ret0, _ := ret[0].(map[verifreg.AllocationId]verifreg.Allocation)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetAllocations indicates an expected call of StateGetAllocations.
-func (mr *MockFullNodeMockRecorder) StateGetAllocations(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetAllocations", reflect.TypeOf((*MockFullNode)(nil).StateGetAllocations), arg0, arg1, arg2)
-}
-
-// StateGetClaim mocks base method.
-func (m *MockFullNode) StateGetClaim(arg0 context.Context, arg1 address.Address, arg2 verifreg.ClaimId, arg3 types.TipSetKey) (*verifreg.Claim, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetClaim", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*verifreg.Claim)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetClaim indicates an expected call of StateGetClaim.
-func (mr *MockFullNodeMockRecorder) StateGetClaim(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaim", reflect.TypeOf((*MockFullNode)(nil).StateGetClaim), arg0, arg1, arg2, arg3)
-}
-
-// StateGetClaims mocks base method.
-func (m *MockFullNode) StateGetClaims(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (map[verifreg.ClaimId]verifreg.Claim, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetClaims", arg0, arg1, arg2)
-	ret0, _ := ret[0].(map[verifreg.ClaimId]verifreg.Claim)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetClaims indicates an expected call of StateGetClaims.
-func (mr *MockFullNodeMockRecorder) StateGetClaims(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetClaims", reflect.TypeOf((*MockFullNode)(nil).StateGetClaims), arg0, arg1, arg2)
-}
-
-// StateGetNetworkParams mocks base method.
-func (m *MockFullNode) StateGetNetworkParams(arg0 context.Context) (*api.NetworkParams, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "StateGetNetworkParams", arg0)
-	ret0, _ := ret[0].(*api.NetworkParams)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// StateGetNetworkParams indicates an expected call of StateGetNetworkParams.
-func (mr *MockFullNodeMockRecorder) StateGetNetworkParams(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateGetNetworkParams", reflect.TypeOf((*MockFullNode)(nil).StateGetNetworkParams), arg0)
-}
-
 // StateGetRandomnessFromBeacon mocks base method.
 func (m *MockFullNode) StateGetRandomnessFromBeacon(arg0 context.Context, arg1 crypto.DomainSeparationTag, arg2 abi.ChainEpoch, arg3 []byte, arg4 types.TipSetKey) (abi.Randomness, error) {
 	m.ctrl.T.Helper()
@@ -2549,10 +2396,10 @@ func (mr *MockFullNodeMockRecorder) StateMarketBalance(arg0, arg1, arg2 interfac
 }
 
 // StateMarketDeals mocks base method.
-func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]*api.MarketDeal, error) {
+func (m *MockFullNode) StateMarketDeals(arg0 context.Context, arg1 types.TipSetKey) (map[string]api.MarketDeal, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMarketDeals", arg0, arg1)
-	ret0, _ := ret[0].(map[string]*api.MarketDeal)
+	ret0, _ := ret[0].(map[string]api.MarketDeal)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2654,10 +2501,10 @@ func (mr *MockFullNodeMockRecorder) StateMinerFaults(arg0, arg1, arg2 interface{
 }
 
 // StateMinerInfo mocks base method.
-func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (api.MinerInfo, error) {
+func (m *MockFullNode) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 types.TipSetKey) (miner.MinerInfo, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInfo", arg0, arg1, arg2)
-	ret0, _ := ret[0].(api.MinerInfo)
+	ret0, _ := ret[0].(miner.MinerInfo)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2669,7 +2516,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerInfo(arg0, arg1, arg2 interface{})
 }
 
 // StateMinerInitialPledgeCollateral mocks base method.
-func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerInitialPledgeCollateral(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerInitialPledgeCollateral", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2714,7 +2561,7 @@ func (mr *MockFullNodeMockRecorder) StateMinerPower(arg0, arg1, arg2 interface{}
 }
 
 // StateMinerPreCommitDepositForPower mocks base method.
-func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
+func (m *MockFullNode) StateMinerPreCommitDepositForPower(arg0 context.Context, arg1 address.Address, arg2 miner0.SectorPreCommitInfo, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateMinerPreCommitDepositForPower", arg0, arg1, arg2, arg3)
 	ret0, _ := ret[0].(big.Int)
@@ -2894,10 +2741,10 @@ func (mr *MockFullNodeMockRecorder) StateSearchMsgLimited(arg0, arg1, arg2 inter
 }
 
 // StateSectorExpiration mocks base method.
-func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorExpiration, error) {
+func (m *MockFullNode) StateSectorExpiration(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorExpiration, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorExpiration", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner0.SectorExpiration)
+	ret0, _ := ret[0].(*miner.SectorExpiration)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
@@ -2924,10 +2771,10 @@ func (mr *MockFullNodeMockRecorder) StateSectorGetInfo(arg0, arg1, arg2, arg3 in
 }
 
 // StateSectorPartition mocks base method.
-func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner0.SectorLocation, error) {
+func (m *MockFullNode) StateSectorPartition(arg0 context.Context, arg1 address.Address, arg2 abi.SectorNumber, arg3 types.TipSetKey) (*miner.SectorLocation, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "StateSectorPartition", arg0, arg1, arg2, arg3)
-	ret0, _ := ret[0].(*miner0.SectorLocation)
+	ret0, _ := ret[0].(*miner.SectorLocation)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }

api/v0api/v1_wrapper.go

@@ -3,40 +3,30 @@ package v0api
 import (
 	"context"
 
-	"github.com/ipfs/go-cid"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
-	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	marketevents "github.com/filecoin-project/lotus/markets/loggers"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/types"
+	"golang.org/x/xerrors"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v1api"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-	marketevents "github.com/filecoin-project/lotus/markets/loggers"
 )
 
 type WrapperV1Full struct {
 	v1api.FullNode
 }
 
-func (w *WrapperV1Full) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
-	pi, err := w.FullNode.StateSectorPreCommitInfo(ctx, maddr, s, tsk)
-	if err != nil {
-		return miner.SectorPreCommitOnChainInfo{}, err
-	}
-	if pi == nil {
-		return miner.SectorPreCommitOnChainInfo{}, xerrors.Errorf("precommit info does not exist")
-	}
-
-	return *pi, nil
-}
-
 func (w *WrapperV1Full) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
 	return w.FullNode.StateSearchMsg(ctx, types.EmptyTSK, msg, api.LookbackNoLimit, true)
 }
@@ -362,8 +352,4 @@ func (w *WrapperV1Full) ClientQueryAsk(ctx context.Context, p peer.ID, miner add
 	return a.Response, nil
 }
 
-func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
-	return w.StateGetBeaconEntry(ctx, epoch)
-}
-
 var _ FullNode = &WrapperV1Full{}

api/v1api/latest.go

@@ -7,10 +7,6 @@ import (
 type FullNode = api.FullNode
 type FullNodeStruct = api.FullNodeStruct
 
-type RawFullNodeAPI FullNode
-
 func PermissionedFullAPI(a FullNode) FullNode {
 	return api.PermissionedFullAPI(a)
 }
-
-type LotusProviderStruct = api.LotusProviderStruct

api/version.go

@@ -3,7 +3,7 @@ package api
 import (
 	"fmt"
 
-	"golang.org/x/xerrors"
+	xerrors "golang.org/x/xerrors"
 )
 
 type Version uint32
@@ -55,12 +55,10 @@ func VersionForType(nodeType NodeType) (Version, error) {
 // semver versions of the rpc api exposed
 var (
 	FullAPIVersion0 = newVer(1, 5, 0)
-	FullAPIVersion1 = newVer(2, 3, 0)
+	FullAPIVersion1 = newVer(2, 2, 0)
 
 	MinerAPIVersion0  = newVer(1, 5, 0)
-	WorkerAPIVersion0 = newVer(1, 7, 0)
+	WorkerAPIVersion0 = newVer(1, 6, 0)
-
-	ProviderAPIVersion0 = newVer(1, 0, 0)
 )
 
 //nolint:varcheck,deadcode

blockstore/api.go

@@ -11,7 +11,6 @@ import (
 type ChainIO interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainPutObj(context.Context, blocks.Block) error
 }
 
 type apiBlockstore struct {
@@ -50,18 +49,12 @@ func (a *apiBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	return len(bb), nil
 }
 
-func (a *apiBlockstore) Put(ctx context.Context, block blocks.Block) error {
+func (a *apiBlockstore) Put(context.Context, blocks.Block) error {
-	return a.api.ChainPutObj(ctx, block)
+	return xerrors.New("not supported")
 }
 
-func (a *apiBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
+func (a *apiBlockstore) PutMany(context.Context, []blocks.Block) error {
-	for _, block := range blocks {
+	return xerrors.New("not supported")
-		err := a.api.ChainPutObj(ctx, block)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 func (a *apiBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {

blockstore/autobatch.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
+	"golang.org/x/xerrors"
+
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	"golang.org/x/xerrors"
 )
 
 // autolog is a logger for the autobatching blockstore. It is subscoped from the
@@ -176,27 +176,23 @@ func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 		return blk, nil
 	}
 
-	if !ipld.IsNotFound(err) {
+	if err != ErrNotFound {
 		return blk, err
 	}
 
 	bs.stateLock.Lock()
+	defer bs.stateLock.Unlock()
 	v, ok := bs.flushingBatch.blockMap[c]
 	if ok {
-		bs.stateLock.Unlock()
 		return v, nil
 	}
 
 	v, ok = bs.bufferedBatch.blockMap[c]
 	if ok {
-		bs.stateLock.Unlock()
 		return v, nil
 	}
-	bs.stateLock.Unlock()
 
-	// We have to check the backing store one more time because it may have been flushed by the
+	return bs.Get(ctx, c)
-	// time we were able to take the lock above.
-	return bs.backingBs.Get(ctx, c)
 }
 
 func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error {
@@ -218,7 +214,7 @@ func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)
 	if err == nil {
 		return true, nil
 	}
-	if ipld.IsNotFound(err) {
+	if err == ErrNotFound {
 		return false, nil
 	}
 

blockstore/autobatch_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/stretchr/testify/require"
 )
 
@@ -30,10 +29,6 @@ func TestAutobatchBlockstore(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, b2.RawData(), v2.RawData())
 
-	// Regression test for a deadlock.
-	_, err = ab.Get(ctx, b3.Cid())
-	require.True(t, ipld.IsNotFound(err))
-
 	require.NoError(t, ab.Flush(ctx))
 	require.NoError(t, ab.Shutdown(ctx))
 }

blockstore/badger/blockstore.go

@@ -13,14 +13,13 @@ import (
 	"github.com/dgraph-io/badger/v2"
 	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/pb"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	logger "github.com/ipfs/go-log/v2"
-	pool "github.com/libp2p/go-buffer-pool"
 	"github.com/multiformats/go-base32"
 	"go.uber.org/zap"
-	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	logger "github.com/ipfs/go-log/v2"
+	pool "github.com/libp2p/go-buffer-pool"
 
 	"github.com/filecoin-project/lotus/blockstore"
 )
@@ -45,8 +44,7 @@ const (
 	// MemoryMap is equivalent to badger/options.MemoryMap.
 	MemoryMap = options.MemoryMap
 	// LoadToRAM is equivalent to badger/options.LoadToRAM.
-	LoadToRAM          = options.LoadToRAM
+	LoadToRAM = options.LoadToRAM
-	defaultGCThreshold = 0.125
 )
 
 // Options embeds the badger options themselves, and augments them with
@@ -396,9 +394,6 @@ func (b *Blockstore) doCopy(from, to *badger.DB) error {
 	if workers < 2 {
 		workers = 2
 	}
-	if workers > 8 {
-		workers = 8
-	}
 
 	stream := from.NewStream()
 	stream.NumGo = workers
@@ -444,7 +439,7 @@ func (b *Blockstore) deleteDB(path string) {
 	}
 }
 
-func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq time.Duration, check func() error) error {
+func (b *Blockstore) onlineGC() error {
 	b.lockDB()
 	defer b.unlockDB()
 
@@ -453,26 +448,14 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
 	if nworkers < 2 {
 		nworkers = 2
 	}
-	if nworkers > 7 { // max out at 1 goroutine per badger level
-		nworkers = 7
-	}
 
 	err := b.db.Flatten(nworkers)
 	if err != nil {
 		return err
 	}
-	checkTick := time.NewTimer(checkFreq)
+
-	defer checkTick.Stop()
 	for err == nil {
-		select {
+		err = b.db.RunValueLogGC(0.125)
-		case <-ctx.Done():
-			err = ctx.Err()
-		case <-checkTick.C:
-			err = check()
-			checkTick.Reset(checkFreq)
-		default:
-			err = b.db.RunValueLogGC(threshold)
-		}
 	}
 
 	if err == badger.ErrNoRewrite {
@@ -485,7 +468,7 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq
 
 // CollectGarbage compacts and runs garbage collection on the value log;
 // implements the BlockstoreGC trait
-func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
+func (b *Blockstore) CollectGarbage(opts ...blockstore.BlockstoreGCOption) error {
 	if err := b.access(); err != nil {
 		return err
 	}
@@ -502,58 +485,8 @@ func (b *Blockstore) CollectGarbage(ctx context.Context, opts ...blockstore.Bloc
 	if options.FullGC {
 		return b.movingGC()
 	}
-	threshold := options.Threshold
-	if threshold == 0 {
-		threshold = defaultGCThreshold
-	}
-	checkFreq := options.CheckFreq
-	if checkFreq < 30*time.Second { // disallow checking more frequently than block time
-		checkFreq = 30 * time.Second
-	}
-	check := options.Check
-	if check == nil {
-		check = func() error {
-			return nil
-		}
-	}
-	return b.onlineGC(ctx, threshold, checkFreq, check)
-}
 
-// GCOnce runs garbage collection on the value log;
+	return b.onlineGC()
-// implements BlockstoreGCOnce trait
-func (b *Blockstore) GCOnce(ctx context.Context, opts ...blockstore.BlockstoreGCOption) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	var options blockstore.BlockstoreGCOptions
-	for _, opt := range opts {
-		err := opt(&options)
-		if err != nil {
-			return err
-		}
-	}
-	if options.FullGC {
-		return xerrors.Errorf("FullGC option specified for GCOnce but full GC is non incremental")
-	}
-
-	threshold := options.Threshold
-	if threshold == 0 {
-		threshold = defaultGCThreshold
-	}
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	// Note no compaction needed before single GC as we will hit at most one vlog anyway
-	err := b.db.RunValueLogGC(threshold)
-	if err == badger.ErrNoRewrite {
-		// not really an error in this case, it signals the end of GC
-		return nil
-	}
-
-	return err
 }
 
 // Size returns the aggregate size of the blockstore
@@ -611,25 +544,13 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro
 		case nil:
 			return item.Value(fn)
 		case badger.ErrKeyNotFound:
-			return ipld.ErrNotFound{Cid: cid}
+			return blockstore.ErrNotFound
 		default:
 			return fmt.Errorf("failed to view block from badger blockstore: %w", err)
 		}
 	})
 }
 
-func (b *Blockstore) Flush(context.Context) error {
-	if err := b.access(); err != nil {
-		return err
-	}
-	defer b.viewers.Done()
-
-	b.lockDB()
-	defer b.unlockDB()
-
-	return b.db.Sync()
-}
-
 // Has implements Blockstore.Has.
 func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	if err := b.access(); err != nil {
@@ -663,7 +584,7 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 // Get implements Blockstore.Get.
 func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
 	if !cid.Defined() {
-		return nil, ipld.ErrNotFound{Cid: cid}
+		return nil, blockstore.ErrNotFound
 	}
 
 	if err := b.access(); err != nil {
@@ -686,7 +607,7 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 			val, err = item.ValueCopy(nil)
 			return err
 		case badger.ErrKeyNotFound:
-			return ipld.ErrNotFound{Cid: cid}
+			return blockstore.ErrNotFound
 		default:
 			return fmt.Errorf("failed to get block from badger blockstore: %w", err)
 		}
@@ -718,7 +639,7 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		case nil:
 			size = int(item.ValueSize())
 		case badger.ErrKeyNotFound:
-			return ipld.ErrNotFound{Cid: cid}
+			return blockstore.ErrNotFound
 		default:
 			return fmt.Errorf("failed to get block size from badger blockstore: %w", err)
 		}
@@ -746,20 +667,6 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error {
 	}
 
 	put := func(db *badger.DB) error {
-		// Check if we have it before writing it.
-		switch err := db.View(func(txn *badger.Txn) error {
-			_, err := txn.Get(k)
-			return err
-		}); err {
-		case badger.ErrKeyNotFound:
-		case nil:
-			// Already exists, skip the put.
-			return nil
-		default:
-			return err
-		}
-
-		// Then write it.
 		err := db.Update(func(txn *badger.Txn) error {
 			return txn.Set(k, block.RawData())
 		})
@@ -815,33 +722,12 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
 		keys = append(keys, k)
 	}
 
-	err := b.db.View(func(txn *badger.Txn) error {
-		for i, k := range keys {
-			switch _, err := txn.Get(k); err {
-			case badger.ErrKeyNotFound:
-			case nil:
-				keys[i] = nil
-			default:
-				// Something is actually wrong
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
 	put := func(db *badger.DB) error {
 		batch := db.NewWriteBatch()
 		defer batch.Cancel()
 
 		for i, block := range blocks {
 			k := keys[i]
-			if k == nil {
-				// skipped because we already have it.
-				continue
-			}
 			if err := batch.Set(k, block.RawData()); err != nil {
 				return err
 			}

blockstore/badger/blockstore_test.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package badgerbs
 
 import (
@@ -10,11 +10,12 @@ import (
 	"strings"
 	"testing"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/lotus/blockstore"
 )
 
@@ -145,7 +146,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 		return nil
 	})
 	g.Go(func() error {
-		return db.CollectGarbage(ctx, blockstore.WithFullGC(true))
+		return db.CollectGarbage(blockstore.WithFullGC(true))
 	})
 
 	err = g.Wait()
@@ -230,7 +231,7 @@ func testMove(t *testing.T, optsF func(string) Options) {
 	checkPath()
 
 	// now do another FullGC to test the double move and following of symlinks
-	if err := db.CollectGarbage(ctx, blockstore.WithFullGC(true)); err != nil {
+	if err := db.CollectGarbage(blockstore.WithFullGC(true)); err != nil {
 		t.Fatal(err)
 	}
 

blockstore/badger/blockstore_test_suite.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package badgerbs
 
 import (
@@ -9,13 +9,13 @@ import (
 	"strings"
 	"testing"
 
-	u "github.com/ipfs/boxo/util"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
+	u "github.com/ipfs/go-ipfs-util"
-	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/blockstore"
+
+	"github.com/stretchr/testify/require"
 )
 
 // TODO: move this to go-ipfs-blockstore.
@@ -56,7 +56,7 @@ func (s *Suite) TestGetWhenKeyNotPresent(t *testing.T) {
 	c := cid.NewCidV0(u.Hash([]byte("stuff")))
 	bl, err := bs.Get(ctx, c)
 	require.Nil(t, bl)
-	require.Equal(t, ipld.ErrNotFound{Cid: c}, err)
+	require.Equal(t, blockstore.ErrNotFound, err)
 }
 
 func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
@@ -69,7 +69,7 @@ func (s *Suite) TestGetWhenKeyIsNil(t *testing.T) {
 	}
 
 	_, err := bs.Get(ctx, cid.Undef)
-	require.Equal(t, ipld.ErrNotFound{Cid: cid.Undef}, err)
+	require.Equal(t, blockstore.ErrNotFound, err)
 }
 
 func (s *Suite) TestPutThenGetBlock(t *testing.T) {
@@ -164,9 +164,8 @@ func (s *Suite) TestPutThenGetSizeBlock(t *testing.T) {
 	require.NoError(t, err)
 	require.Zero(t, emptySize)
 
-	missingCid := missingBlock.Cid()
+	missingSize, err := bs.GetSize(ctx, missingBlock.Cid())
-	missingSize, err := bs.GetSize(ctx, missingCid)
+	require.Equal(t, blockstore.ErrNotFound, err)
-	require.Equal(t, ipld.ErrNotFound{Cid: missingCid}, err)
 	require.Equal(t, -1, missingSize)
 }
 

blockstore/blockstore.go

@@ -2,16 +2,18 @@ package blockstore
 
 import (
 	"context"
-	"time"
 
-	blockstore "github.com/ipfs/boxo/blockstore"
+	cid "github.com/ipfs/go-cid"
-	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
+
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
 var log = logging.Logger("blockstore")
 
+var ErrNotFound = blockstore.ErrNotFound
+
 // Blockstore is the blockstore interface used by Lotus. It is the union
 // of the basic go-ipfs blockstore, with other capabilities required by Lotus,
 // e.g. View or Sync.
@@ -19,7 +21,6 @@ type Blockstore interface {
 	blockstore.Blockstore
 	blockstore.Viewer
 	BatchDeleter
-	Flusher
 }
 
 // BasicBlockstore is an alias to the original IPFS Blockstore.
@@ -27,10 +28,6 @@ type BasicBlockstore = blockstore.Blockstore
 
 type Viewer = blockstore.Viewer
 
-type Flusher interface {
-	Flush(context.Context) error
-}
-
 type BatchDeleter interface {
 	DeleteMany(ctx context.Context, cids []cid.Cid) error
 }
@@ -42,12 +39,7 @@ type BlockstoreIterator interface {
 
 // BlockstoreGC is a trait for blockstores that support online garbage collection
 type BlockstoreGC interface {
-	CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error
+	CollectGarbage(options ...BlockstoreGCOption) error
-}
-
-// BlockstoreGCOnce is a trait for a blockstore that supports incremental online garbage collection
-type BlockstoreGCOnce interface {
-	GCOnce(ctx context.Context, options ...BlockstoreGCOption) error
 }
 
 // BlockstoreGCOption is a functional interface for controlling blockstore GC options
@@ -56,12 +48,6 @@ type BlockstoreGCOption = func(*BlockstoreGCOptions) error
 // BlockstoreGCOptions is a struct with GC options
 type BlockstoreGCOptions struct {
 	FullGC bool
-	// fraction of garbage in badger vlog before its worth processing in online GC
-	Threshold float64
-	// how often to call the check function
-	CheckFreq time.Duration
-	// function to call periodically to pause or early terminate GC
-	Check func() error
 }
 
 func WithFullGC(fullgc bool) BlockstoreGCOption {
@@ -71,27 +57,6 @@ func WithFullGC(fullgc bool) BlockstoreGCOption {
 	}
 }
 
-func WithThreshold(threshold float64) BlockstoreGCOption {
-	return func(opts *BlockstoreGCOptions) error {
-		opts.Threshold = threshold
-		return nil
-	}
-}
-
-func WithCheckFreq(f time.Duration) BlockstoreGCOption {
-	return func(opts *BlockstoreGCOptions) error {
-		opts.CheckFreq = f
-		return nil
-	}
-}
-
-func WithCheck(check func() error) BlockstoreGCOption {
-	return func(opts *BlockstoreGCOptions) error {
-		opts.Check = check
-		return nil
-	}
-}
-
 // BlockstoreSize is a trait for on-disk blockstores that can report their size
 type BlockstoreSize interface {
 	Size() (int64, error)
@@ -130,13 +95,6 @@ type adaptedBlockstore struct {
 
 var _ Blockstore = (*adaptedBlockstore)(nil)
 
-func (a *adaptedBlockstore) Flush(ctx context.Context) error {
-	if flusher, canFlush := a.Blockstore.(Flusher); canFlush {
-		return flusher.Flush(ctx)
-	}
-	return nil
-}
-
 func (a *adaptedBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error {
 	blk, err := a.Get(ctx, cid)
 	if err != nil {

blockstore/buffered.go

@@ -6,7 +6,6 @@ import (
 
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // buflog is a logger for the buffered blockstore. It is subscoped from the
@@ -46,8 +45,6 @@ var (
 	_ Viewer     = (*BufferedBlockstore)(nil)
 )
 
-func (bs *BufferedBlockstore) Flush(ctx context.Context) error { return bs.write.Flush(ctx) }
-
 func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	a, err := bs.read.AllKeysChan(ctx)
 	if err != nil {
@@ -109,7 +106,7 @@ func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) er
 
 func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
 	// both stores are viewable.
-	if err := bs.write.View(ctx, c, callback); ipld.IsNotFound(err) {
+	if err := bs.write.View(ctx, c, callback); err == ErrNotFound {
 		// not found in write blockstore; fall through.
 	} else {
 		return err // propagate errors, or nil, i.e. found.
@@ -119,7 +116,7 @@ func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func
 
 func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) {
 	if out, err := bs.write.Get(ctx, c); err != nil {
-		if !ipld.IsNotFound(err) {
+		if err != ErrNotFound {
 			return nil, err
 		}
 	} else {
@@ -131,7 +128,7 @@ func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block,
 
 func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	s, err := bs.read.GetSize(ctx, c)
-	if ipld.IsNotFound(err) || s == 0 {
+	if err == ErrNotFound || s == 0 {
 		return bs.write.GetSize(ctx, c)
 	}
 

blockstore/cbor_gen.go

@@ -1,459 +0,0 @@
-// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
-
-package blockstore
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"sort"
-
-	cid "github.com/ipfs/go-cid"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	xerrors "golang.org/x/xerrors"
-)
-
-var _ = xerrors.Errorf
-var _ = cid.Undef
-var _ = math.E
-var _ = sort.Sort
-
-var lengthBufNetRpcReq = []byte{132}
-
-func (t *NetRpcReq) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-
-	cw := cbg.NewCborWriter(w)
-
-	if _, err := cw.Write(lengthBufNetRpcReq); err != nil {
-		return err
-	}
-
-	// t.Type (blockstore.NetRPCReqType) (uint8)
-	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
-		return err
-	}
-
-	// t.ID (uint64) (uint64)
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
-		return err
-	}
-
-	// t.Cid ([]cid.Cid) (slice)
-	if len(t.Cid) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.Cid was too long")
-	}
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Cid))); err != nil {
-		return err
-	}
-	for _, v := range t.Cid {
-
-		if err := cbg.WriteCid(cw, v); err != nil {
-			return xerrors.Errorf("failed to write cid field v: %w", err)
-		}
-
-	}
-
-	// t.Data ([][]uint8) (slice)
-	if len(t.Data) > cbg.MaxLength {
-		return xerrors.Errorf("Slice value in field t.Data was too long")
-	}
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Data))); err != nil {
-		return err
-	}
-	for _, v := range t.Data {
-		if len(v) > cbg.ByteArrayMaxLen {
-			return xerrors.Errorf("Byte array in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(v))); err != nil {
-			return err
-		}
-
-		if _, err := cw.Write(v[:]); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (t *NetRpcReq) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = NetRpcReq{}
-
-	cr := cbg.NewCborReader(r)
-
-	maj, extra, err := cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err == io.EOF {
-			err = io.ErrUnexpectedEOF
-		}
-	}()
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 4 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Type (blockstore.NetRPCReqType) (uint8)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint8 field")
-	}
-	if extra > math.MaxUint8 {
-		return fmt.Errorf("integer in input was too large for uint8 field")
-	}
-	t.Type = NetRPCReqType(extra)
-	// t.ID (uint64) (uint64)
-
-	{
-
-		maj, extra, err = cr.ReadHeader()
-		if err != nil {
-			return err
-		}
-		if maj != cbg.MajUnsignedInt {
-			return fmt.Errorf("wrong type for uint64 field")
-		}
-		t.ID = uint64(extra)
-
-	}
-	// t.Cid ([]cid.Cid) (slice)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Cid: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.Cid = make([]cid.Cid, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-		{
-			var maj byte
-			var extra uint64
-			var err error
-			_ = maj
-			_ = extra
-			_ = err
-
-			{
-
-				c, err := cbg.ReadCid(cr)
-				if err != nil {
-					return xerrors.Errorf("failed to read cid field t.Cid[i]: %w", err)
-				}
-
-				t.Cid[i] = c
-
-			}
-		}
-	}
-
-	// t.Data ([][]uint8) (slice)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Data: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-
-	if extra > 0 {
-		t.Data = make([][]uint8, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-		{
-			var maj byte
-			var extra uint64
-			var err error
-			_ = maj
-			_ = extra
-			_ = err
-
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
-
-			if extra > cbg.ByteArrayMaxLen {
-				return fmt.Errorf("t.Data[i]: byte array too large (%d)", extra)
-			}
-			if maj != cbg.MajByteString {
-				return fmt.Errorf("expected byte array")
-			}
-
-			if extra > 0 {
-				t.Data[i] = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(cr, t.Data[i][:]); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-var lengthBufNetRpcResp = []byte{131}
-
-func (t *NetRpcResp) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-
-	cw := cbg.NewCborWriter(w)
-
-	if _, err := cw.Write(lengthBufNetRpcResp); err != nil {
-		return err
-	}
-
-	// t.Type (blockstore.NetRPCRespType) (uint8)
-	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
-		return err
-	}
-
-	// t.ID (uint64) (uint64)
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ID)); err != nil {
-		return err
-	}
-
-	// t.Data ([]uint8) (slice)
-	if len(t.Data) > cbg.ByteArrayMaxLen {
-		return xerrors.Errorf("Byte array in field t.Data was too long")
-	}
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Data))); err != nil {
-		return err
-	}
-
-	if _, err := cw.Write(t.Data[:]); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *NetRpcResp) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = NetRpcResp{}
-
-	cr := cbg.NewCborReader(r)
-
-	maj, extra, err := cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err == io.EOF {
-			err = io.ErrUnexpectedEOF
-		}
-	}()
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 3 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Type (blockstore.NetRPCRespType) (uint8)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint8 field")
-	}
-	if extra > math.MaxUint8 {
-		return fmt.Errorf("integer in input was too large for uint8 field")
-	}
-	t.Type = NetRPCRespType(extra)
-	// t.ID (uint64) (uint64)
-
-	{
-
-		maj, extra, err = cr.ReadHeader()
-		if err != nil {
-			return err
-		}
-		if maj != cbg.MajUnsignedInt {
-			return fmt.Errorf("wrong type for uint64 field")
-		}
-		t.ID = uint64(extra)
-
-	}
-	// t.Data ([]uint8) (slice)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.ByteArrayMaxLen {
-		return fmt.Errorf("t.Data: byte array too large (%d)", extra)
-	}
-	if maj != cbg.MajByteString {
-		return fmt.Errorf("expected byte array")
-	}
-
-	if extra > 0 {
-		t.Data = make([]uint8, extra)
-	}
-
-	if _, err := io.ReadFull(cr, t.Data[:]); err != nil {
-		return err
-	}
-	return nil
-}
-
-var lengthBufNetRpcErr = []byte{131}
-
-func (t *NetRpcErr) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-
-	cw := cbg.NewCborWriter(w)
-
-	if _, err := cw.Write(lengthBufNetRpcErr); err != nil {
-		return err
-	}
-
-	// t.Type (blockstore.NetRPCErrType) (uint8)
-	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Type)); err != nil {
-		return err
-	}
-
-	// t.Msg (string) (string)
-	if len(t.Msg) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Msg was too long")
-	}
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Msg))); err != nil {
-		return err
-	}
-	if _, err := cw.WriteString(string(t.Msg)); err != nil {
-		return err
-	}
-
-	// t.Cid (cid.Cid) (struct)
-
-	if t.Cid == nil {
-		if _, err := cw.Write(cbg.CborNull); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteCid(cw, *t.Cid); err != nil {
-			return xerrors.Errorf("failed to write cid field t.Cid: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (t *NetRpcErr) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = NetRpcErr{}
-
-	cr := cbg.NewCborReader(r)
-
-	maj, extra, err := cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err == io.EOF {
-			err = io.ErrUnexpectedEOF
-		}
-	}()
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 3 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Type (blockstore.NetRPCErrType) (uint8)
-
-	maj, extra, err = cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint8 field")
-	}
-	if extra > math.MaxUint8 {
-		return fmt.Errorf("integer in input was too large for uint8 field")
-	}
-	t.Type = NetRPCErrType(extra)
-	// t.Msg (string) (string)
-
-	{
-		sval, err := cbg.ReadString(cr)
-		if err != nil {
-			return err
-		}
-
-		t.Msg = string(sval)
-	}
-	// t.Cid (cid.Cid) (struct)
-
-	{
-
-		b, err := cr.ReadByte()
-		if err != nil {
-			return err
-		}
-		if b != cbg.CborNull[0] {
-			if err := cr.UnreadByte(); err != nil {
-				return err
-			}
-
-			c, err := cbg.ReadCid(cr)
-			if err != nil {
-				return xerrors.Errorf("failed to read cid field t.Cid: %w", err)
-			}
-
-			t.Cid = &c
-		}
-
-	}
-	return nil
-}

blockstore/discard.go

@@ -5,7 +5,7 @@ import (
 	"io"
 
 	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
+	cid "github.com/ipfs/go-cid"
 )
 
 var _ Blockstore = (*discardstore)(nil)
@@ -38,10 +38,6 @@ func (b *discardstore) View(ctx context.Context, cid cid.Cid, f func([]byte) err
 	return b.bs.View(ctx, cid, f)
 }
 
-func (b *discardstore) Flush(ctx context.Context) error {
-	return nil
-}
-
 func (b *discardstore) Put(ctx context.Context, blk blocks.Block) error {
 	return nil
 }

blockstore/fallback.go

@@ -5,10 +5,10 @@ import (
 	"sync"
 	"time"
 
+	"golang.org/x/xerrors"
+
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	"golang.org/x/xerrors"
 )
 
 // UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore
@@ -56,7 +56,7 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 		if fbs.missFn == nil {
 			log.Errorw("fallbackstore: missFn not configured yet")
-			return nil, ipld.ErrNotFound{Cid: c}
+			return nil, ErrNotFound
 		}
 	}
 
@@ -79,10 +79,10 @@ func (fbs *FallbackStore) getFallback(c cid.Cid) (blocks.Block, error) {
 
 func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
 	b, err := fbs.Blockstore.Get(ctx, c)
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		return b, nil
-	case ipld.IsNotFound(err):
+	case ErrNotFound:
 		return fbs.getFallback(c)
 	default:
 		return b, err
@@ -91,10 +91,10 @@ func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, err
 
 func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
 	sz, err := fbs.Blockstore.GetSize(ctx, c)
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		return sz, nil
-	case ipld.IsNotFound(err):
+	case ErrNotFound:
 		b, err := fbs.getFallback(c)
 		if err != nil {
 			return 0, err

blockstore/idstore.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"io"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
 )
 
 var _ Blockstore = (*idstore)(nil)
@@ -102,14 +103,6 @@ func (b *idstore) Put(ctx context.Context, blk blocks.Block) error {
 	return b.bs.Put(ctx, blk)
 }
 
-func (b *idstore) ForEachKey(f func(cid.Cid) error) error {
-	iterBstore, ok := b.bs.(BlockstoreIterator)
-	if !ok {
-		return xerrors.Errorf("underlying blockstore (type %T) doesn't support fast iteration", b.bs)
-	}
-	return iterBstore.ForEachKey(f)
-}
-
 func (b *idstore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	toPut := make([]blocks.Block, 0, len(blks))
 	for _, blk := range blks {
@@ -179,21 +172,3 @@ func (b *idstore) Close() error {
 	}
 	return nil
 }
-
-func (b *idstore) Flush(ctx context.Context) error {
-	return b.bs.Flush(ctx)
-}
-
-func (b *idstore) CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error {
-	if bs, ok := b.bs.(BlockstoreGC); ok {
-		return bs.CollectGarbage(ctx, options...)
-	}
-	return xerrors.Errorf("not supported")
-}
-
-func (b *idstore) GCOnce(ctx context.Context, options ...BlockstoreGCOption) error {
-	if bs, ok := b.bs.(BlockstoreGCOnce); ok {
-		return bs.GCOnce(ctx, options...)
-	}
-	return xerrors.Errorf("not supported")
-}

blockstore/ipfs.go

@@ -3,18 +3,19 @@ package blockstore
 import (
 	"bytes"
 	"context"
-	"io"
+	"io/ioutil"
 
-	iface "github.com/ipfs/boxo/coreiface"
-	"github.com/ipfs/boxo/coreiface/options"
-	"github.com/ipfs/boxo/coreiface/path"
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multiaddr"
-	"github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
 
-	rpc "github.com/filecoin-project/kubo-api-client"
+	"github.com/multiformats/go-multiaddr"
+	"github.com/multiformats/go-multihash"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/interface-go-ipfs-core/options"
+	"github.com/ipfs/interface-go-ipfs-core/path"
 )
 
 type IPFSBlockstore struct {
@@ -25,7 +26,7 @@ type IPFSBlockstore struct {
 var _ BasicBlockstore = (*IPFSBlockstore)(nil)
 
 func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error) {
-	localApi, err := rpc.NewLocalApi()
+	localApi, err := httpapi.NewLocalApi()
 	if err != nil {
 		return nil, xerrors.Errorf("getting local ipfs api: %w", err)
 	}
@@ -52,7 +53,7 @@ func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, e
 }
 
 func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error) {
-	httpApi, err := rpc.NewApi(maddr)
+	httpApi, err := httpapi.NewApi(maddr)
 	if err != nil {
 		return nil, xerrors.Errorf("setting remote ipfs api: %w", err)
 	}
@@ -104,7 +105,7 @@ func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, er
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
 	}
 
-	data, err := io.ReadAll(rd)
+	data, err := ioutil.ReadAll(rd)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +130,7 @@ func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error {
 
 	_, err = i.api.Block().Put(ctx, bytes.NewReader(block.RawData()),
 		options.Block.Hash(mhd.Code, mhd.Length),
-		options.Block.Format(multihash.Codes[block.Cid().Type()]))
+		options.Block.Format(cid.CodecToStr[block.Cid().Type()]))
 	return err
 }
 

blockstore/mem.go

@@ -5,7 +5,6 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 )
 
 // NewMemory returns a temporary memory-backed blockstore.
@@ -14,52 +13,46 @@ func NewMemory() MemBlockstore {
 }
 
 // MemBlockstore is a terminal blockstore that keeps blocks in memory.
-// To match behavior of badger blockstore we index by multihash only.
+type MemBlockstore map[cid.Cid]blocks.Block
-type MemBlockstore map[string]blocks.Block
-
-func (MemBlockstore) Flush(context.Context) error { return nil }
 
 func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
-	delete(m, string(k.Hash()))
+	delete(m, k)
 	return nil
 }
 
 func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error {
 	for _, k := range ks {
-		delete(m, string(k.Hash()))
+		delete(m, k)
 	}
 	return nil
 }
 
 func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error) {
-	_, ok := m[string(k.Hash())]
+	_, ok := m[k]
 	return ok, nil
 }
 
 func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error {
-	b, ok := m[string(k.Hash())]
+	b, ok := m[k]
 	if !ok {
-		return ipld.ErrNotFound{Cid: k}
+		return ErrNotFound
 	}
 	return callback(b.RawData())
 }
 
 func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) {
-	b, ok := m[string(k.Hash())]
+	b, ok := m[k]
 	if !ok {
-		return nil, ipld.ErrNotFound{Cid: k}
+		return nil, ErrNotFound
-	}
-	if b.Cid().Prefix().Codec != k.Prefix().Codec {
-		return blocks.NewBlockWithCid(b.RawData(), k)
 	}
 	return b, nil
 }
 
 // GetSize returns the CIDs mapped BlockSize
 func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
-	b, ok := m[string(k.Hash())]
+	b, ok := m[k]
 	if !ok {
-		return 0, ipld.ErrNotFound{Cid: k}
+		return 0, ErrNotFound
 	}
 	return len(b.RawData()), nil
 }
@@ -68,7 +61,7 @@ func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error) {
 func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Convert to a basic block for safety, but try to reuse the existing
 	// block if it's already a basic block.
-	k := string(b.Cid().Hash())
+	k := b.Cid()
 	if _, ok := b.(*blocks.BasicBlock); !ok {
 		// If we already have the block, abort.
 		if _, ok := m[k]; ok {
@@ -77,7 +70,7 @@ func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error {
 		// the error is only for debugging.
 		b, _ = blocks.NewBlockWithCid(b.RawData(), b.Cid())
 	}
-	m[k] = b
+	m[b.Cid()] = b
 	return nil
 }
 
@@ -95,8 +88,8 @@ func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
 // the given context, closing the channel if it becomes Done.
 func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	ch := make(chan cid.Cid, len(m))
-	for _, b := range m {
+	for k := range m {
-		ch <- b.Cid()
+		ch <- k
 	}
 	close(ch)
 	return ch, nil

blockstore/mem_test.go

@@ -1,45 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"testing"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMemGetCodec(t *testing.T) {
-	ctx := context.Background()
-	bs := NewMemory()
-
-	cborArr := []byte{0x82, 1, 2}
-
-	h, err := mh.Sum(cborArr, mh.SHA2_256, -1)
-	require.NoError(t, err)
-
-	rawCid := cid.NewCidV1(cid.Raw, h)
-	rawBlk, err := blocks.NewBlockWithCid(cborArr, rawCid)
-	require.NoError(t, err)
-
-	err = bs.Put(ctx, rawBlk)
-	require.NoError(t, err)
-
-	cborCid := cid.NewCidV1(cid.DagCBOR, h)
-
-	cborBlk, err := bs.Get(ctx, cborCid)
-	require.NoError(t, err)
-
-	require.Equal(t, cborCid.Prefix(), cborBlk.Cid().Prefix())
-	require.EqualValues(t, cborArr, cborBlk.RawData())
-
-	// was allocated
-	require.NotEqual(t, cborBlk, rawBlk)
-
-	gotRawBlk, err := bs.Get(ctx, rawCid)
-	require.NoError(t, err)
-
-	// not allocated
-	require.Equal(t, rawBlk, gotRawBlk)
-}

blockstore/net.go

@@ -1,426 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"fmt"
-	"sync"
-	"sync/atomic"
-
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	"github.com/libp2p/go-msgio"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	"golang.org/x/xerrors"
-)
-
-type NetRPCReqType byte
-
-const (
-	NRpcHas NetRPCReqType = iota
-	NRpcGet
-	NRpcGetSize
-	NRpcPut
-	NRpcDelete
-
-	// todo cancel req
-)
-
-type NetRPCRespType byte
-
-const (
-	NRpcOK NetRPCRespType = iota
-	NRpcErr
-	NRpcMore
-)
-
-type NetRPCErrType byte
-
-const (
-	NRpcErrGeneric NetRPCErrType = iota
-	NRpcErrNotFound
-)
-
-type NetRpcReq struct {
-	Type NetRPCReqType
-	ID   uint64
-
-	Cid  []cid.Cid // todo maxsize?
-	Data [][]byte  // todo maxsize?
-}
-
-type NetRpcResp struct {
-	Type NetRPCRespType
-	ID   uint64
-
-	// error or cids in allkeys
-	Data []byte // todo maxsize?
-
-	next <-chan NetRpcResp
-}
-
-type NetRpcErr struct {
-	Type NetRPCErrType
-
-	Msg string
-
-	// in case of NRpcErrNotFound
-	Cid *cid.Cid
-}
-
-type NetworkStore struct {
-	// note: writer is thread-safe
-	msgStream msgio.ReadWriteCloser
-
-	// atomic
-	reqCount uint64
-
-	respLk sync.Mutex
-
-	// respMap is nil after store closes
-	respMap map[uint64]chan<- NetRpcResp
-
-	closing chan struct{}
-	closed  chan struct{}
-
-	closeLk sync.Mutex
-	onClose []func()
-}
-
-func NewNetworkStore(mss msgio.ReadWriteCloser) *NetworkStore {
-	ns := &NetworkStore{
-		msgStream: mss,
-
-		respMap: map[uint64]chan<- NetRpcResp{},
-
-		closing: make(chan struct{}),
-		closed:  make(chan struct{}),
-	}
-
-	go ns.receive()
-
-	return ns
-}
-
-func (n *NetworkStore) shutdown(msg string) {
-	if err := n.msgStream.Close(); err != nil {
-		log.Errorw("closing netstore msg stream", "error", err)
-	}
-
-	nerr := NetRpcErr{
-		Type: NRpcErrGeneric,
-		Msg:  msg,
-		Cid:  nil,
-	}
-
-	var errb bytes.Buffer
-	if err := nerr.MarshalCBOR(&errb); err != nil {
-		log.Errorw("netstore shutdown: error marshaling error", "err", err)
-	}
-
-	n.respLk.Lock()
-	for id, resps := range n.respMap {
-		resps <- NetRpcResp{
-			Type: NRpcErr,
-			ID:   id,
-			Data: errb.Bytes(),
-		}
-	}
-
-	n.respMap = nil
-
-	n.respLk.Unlock()
-}
-
-func (n *NetworkStore) OnClose(cb func()) {
-	n.closeLk.Lock()
-	defer n.closeLk.Unlock()
-
-	select {
-	case <-n.closed:
-		cb()
-	default:
-		n.onClose = append(n.onClose, cb)
-	}
-}
-
-func (n *NetworkStore) receive() {
-	defer func() {
-		n.closeLk.Lock()
-		defer n.closeLk.Unlock()
-
-		close(n.closed)
-		if n.onClose != nil {
-			for _, f := range n.onClose {
-				f()
-			}
-		}
-	}()
-
-	for {
-		select {
-		case <-n.closing:
-			n.shutdown("netstore stopping")
-			return
-		default:
-		}
-
-		msg, err := n.msgStream.ReadMsg()
-		if err != nil {
-			n.shutdown(fmt.Sprintf("netstore ReadMsg: %s", err))
-			return
-		}
-
-		var resp NetRpcResp
-		if err := resp.UnmarshalCBOR(bytes.NewReader(msg)); err != nil {
-			n.shutdown(fmt.Sprintf("unmarshaling netstore response: %s", err))
-			return
-		}
-
-		n.msgStream.ReleaseMsg(msg)
-
-		n.respLk.Lock()
-		if ch, ok := n.respMap[resp.ID]; ok {
-			if resp.Type == NRpcMore {
-				nch := make(chan NetRpcResp, 1)
-				resp.next = nch
-				n.respMap[resp.ID] = nch
-			} else {
-				delete(n.respMap, resp.ID)
-			}
-
-			ch <- resp
-		}
-		n.respLk.Unlock()
-	}
-}
-
-func (n *NetworkStore) sendRpc(rt NetRPCReqType, cids []cid.Cid, data [][]byte) (uint64, <-chan NetRpcResp, error) {
-	rid := atomic.AddUint64(&n.reqCount, 1)
-
-	respCh := make(chan NetRpcResp, 1) // todo pool?
-
-	n.respLk.Lock()
-	if n.respMap == nil {
-		n.respLk.Unlock()
-		return 0, nil, xerrors.Errorf("netstore closed")
-	}
-	n.respMap[rid] = respCh
-	n.respLk.Unlock()
-
-	req := NetRpcReq{
-		Type: rt,
-		ID:   rid,
-		Cid:  cids,
-		Data: data,
-	}
-
-	var rbuf bytes.Buffer // todo buffer pool
-	if err := req.MarshalCBOR(&rbuf); err != nil {
-		n.respLk.Lock()
-		defer n.respLk.Unlock()
-
-		if n.respMap == nil {
-			return 0, nil, xerrors.Errorf("netstore closed")
-		}
-		delete(n.respMap, rid)
-
-		return 0, nil, err
-	}
-
-	if err := n.msgStream.WriteMsg(rbuf.Bytes()); err != nil {
-		n.respLk.Lock()
-		defer n.respLk.Unlock()
-
-		if n.respMap == nil {
-			return 0, nil, xerrors.Errorf("netstore closed")
-		}
-		delete(n.respMap, rid)
-
-		return 0, nil, err
-	}
-
-	return rid, respCh, nil
-}
-
-func (n *NetworkStore) waitResp(ctx context.Context, rch <-chan NetRpcResp, rid uint64) (NetRpcResp, error) {
-	select {
-	case resp := <-rch:
-		if resp.Type == NRpcErr {
-			var e NetRpcErr
-			if err := e.UnmarshalCBOR(bytes.NewReader(resp.Data)); err != nil {
-				return NetRpcResp{}, xerrors.Errorf("unmarshaling error data: %w", err)
-			}
-
-			var err error
-			switch e.Type {
-			case NRpcErrNotFound:
-				if e.Cid != nil {
-					err = ipld.ErrNotFound{
-						Cid: *e.Cid,
-					}
-				} else {
-					err = xerrors.Errorf("block not found, but cid was null")
-				}
-			case NRpcErrGeneric:
-				err = xerrors.Errorf("generic error")
-			default:
-				err = xerrors.Errorf("unknown error type")
-			}
-
-			return NetRpcResp{}, xerrors.Errorf("netstore error response: %s (%w)", e.Msg, err)
-		}
-
-		return resp, nil
-	case <-ctx.Done():
-		// todo send cancel req
-
-		n.respLk.Lock()
-		if n.respMap != nil {
-			delete(n.respMap, rid)
-		}
-		n.respLk.Unlock()
-
-		return NetRpcResp{}, ctx.Err()
-	}
-}
-
-func (n *NetworkStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
-	req, rch, err := n.sendRpc(NRpcHas, []cid.Cid{c}, nil)
-	if err != nil {
-		return false, err
-	}
-
-	resp, err := n.waitResp(ctx, rch, req)
-	if err != nil {
-		return false, err
-	}
-
-	if len(resp.Data) != 1 {
-		return false, xerrors.Errorf("expected reposnse length to be 1 byte")
-	}
-	switch resp.Data[0] {
-	case cbg.CborBoolTrue[0]:
-		return true, nil
-	case cbg.CborBoolFalse[0]:
-		return false, nil
-	default:
-		return false, xerrors.Errorf("has: bad response: %x", resp.Data[0])
-	}
-}
-
-func (n *NetworkStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
-	req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := n.waitResp(ctx, rch, req)
-	if err != nil {
-		return nil, err
-	}
-
-	return blocks.NewBlockWithCid(resp.Data, c)
-}
-
-func (n *NetworkStore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error {
-	req, rch, err := n.sendRpc(NRpcGet, []cid.Cid{c}, nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := n.waitResp(ctx, rch, req)
-	if err != nil {
-		return err
-	}
-
-	return callback(resp.Data) // todo return buf to pool
-}
-
-func (n *NetworkStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
-	req, rch, err := n.sendRpc(NRpcGetSize, []cid.Cid{c}, nil)
-	if err != nil {
-		return 0, err
-	}
-
-	resp, err := n.waitResp(ctx, rch, req)
-	if err != nil {
-		return 0, err
-	}
-
-	if len(resp.Data) != 4 {
-		return 0, xerrors.Errorf("expected getsize response to be 4 bytes, was %d", resp.Data)
-	}
-
-	return int(binary.LittleEndian.Uint32(resp.Data)), nil
-}
-
-func (n *NetworkStore) Put(ctx context.Context, block blocks.Block) error {
-	return n.PutMany(ctx, []blocks.Block{block})
-}
-
-func (n *NetworkStore) PutMany(ctx context.Context, blocks []blocks.Block) error {
-	// todo pool
-	cids := make([]cid.Cid, len(blocks))
-	blkDatas := make([][]byte, len(blocks))
-	for i, block := range blocks {
-		cids[i] = block.Cid()
-		blkDatas[i] = block.RawData()
-	}
-
-	req, rch, err := n.sendRpc(NRpcPut, cids, blkDatas)
-	if err != nil {
-		return err
-	}
-
-	_, err = n.waitResp(ctx, rch, req)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (n *NetworkStore) DeleteBlock(ctx context.Context, c cid.Cid) error {
-	return n.DeleteMany(ctx, []cid.Cid{c})
-}
-
-func (n *NetworkStore) DeleteMany(ctx context.Context, cids []cid.Cid) error {
-	req, rch, err := n.sendRpc(NRpcDelete, cids, nil)
-	if err != nil {
-		return err
-	}
-
-	_, err = n.waitResp(ctx, rch, req)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (n *NetworkStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	return nil, xerrors.Errorf("not supported")
-}
-
-func (n *NetworkStore) HashOnRead(enabled bool) {
-	// todo
-	return
-}
-
-func (*NetworkStore) Flush(context.Context) error { return nil }
-
-func (n *NetworkStore) Stop(ctx context.Context) error {
-	close(n.closing)
-
-	select {
-	case <-n.closed:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}
-
-var _ Blockstore = &NetworkStore{}

blockstore/net_serve.go

@@ -1,237 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-
-	block "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	"github.com/libp2p/go-msgio"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	"golang.org/x/xerrors"
-)
-
-type NetworkStoreHandler struct {
-	msgStream msgio.ReadWriteCloser
-
-	bs Blockstore
-}
-
-// NOTE: This code isn't yet hardened to accept untrusted input. See TODOs here and in net.go
-func HandleNetBstoreStream(ctx context.Context, bs Blockstore, mss msgio.ReadWriteCloser) *NetworkStoreHandler {
-	ns := &NetworkStoreHandler{
-		msgStream: mss,
-		bs:        bs,
-	}
-
-	go ns.handle(ctx)
-
-	return ns
-}
-
-func (h *NetworkStoreHandler) handle(ctx context.Context) {
-	defer func() {
-		if err := h.msgStream.Close(); err != nil {
-			log.Errorw("error closing blockstore stream", "error", err)
-		}
-	}()
-
-	for {
-		var req NetRpcReq
-
-		ms, err := h.msgStream.ReadMsg()
-		if err != nil {
-			log.Warnw("bstore stream err", "error", err)
-			return
-		}
-
-		if err := req.UnmarshalCBOR(bytes.NewReader(ms)); err != nil {
-			return
-		}
-
-		h.msgStream.ReleaseMsg(ms)
-
-		switch req.Type {
-		case NRpcHas:
-			if len(req.Cid) != 1 {
-				if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			res, err := h.bs.Has(ctx, req.Cid[0])
-			if err != nil {
-				if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			var resData [1]byte
-			if res {
-				resData[0] = cbg.CborBoolTrue[0]
-			} else {
-				resData[0] = cbg.CborBoolFalse[0]
-			}
-
-			if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
-				log.Warnw("writing response", "error", err)
-				return
-			}
-
-		case NRpcGet:
-			if len(req.Cid) != 1 {
-				if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			err := h.bs.View(ctx, req.Cid[0], func(bdata []byte) error {
-				return h.respond(req.ID, NRpcOK, bdata)
-			})
-			if err != nil {
-				if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-		case NRpcGetSize:
-			if len(req.Cid) != 1 {
-				if err := h.respondError(req.ID, xerrors.New("expected request for 1 cid"), cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			sz, err := h.bs.GetSize(ctx, req.Cid[0])
-			if err != nil {
-				if err := h.respondError(req.ID, err, req.Cid[0]); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			var resData [4]byte
-			binary.LittleEndian.PutUint32(resData[:], uint32(sz))
-
-			if err := h.respond(req.ID, NRpcOK, resData[:]); err != nil {
-				log.Warnw("writing response", "error", err)
-				return
-			}
-
-		case NRpcPut:
-			blocks := make([]block.Block, len(req.Cid))
-
-			if len(req.Cid) != len(req.Data) {
-				if err := h.respondError(req.ID, xerrors.New("cid count didn't match data count"), cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-				}
-				return
-			}
-
-			for i := range req.Cid {
-				blocks[i], err = block.NewBlockWithCid(req.Data[i], req.Cid[i])
-				if err != nil {
-					log.Warnw("make block", "error", err)
-					return
-				}
-			}
-
-			err := h.bs.PutMany(ctx, blocks)
-			if err != nil {
-				if err := h.respondError(req.ID, err, cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
-				log.Warnw("writing response", "error", err)
-				return
-			}
-		case NRpcDelete:
-			err := h.bs.DeleteMany(ctx, req.Cid)
-			if err != nil {
-				if err := h.respondError(req.ID, err, cid.Undef); err != nil {
-					log.Warnw("writing error response", "error", err)
-					return
-				}
-				continue
-			}
-
-			if err := h.respond(req.ID, NRpcOK, []byte{}); err != nil {
-				log.Warnw("writing response", "error", err)
-				return
-			}
-		default:
-			if err := h.respondError(req.ID, xerrors.New("unsupported request type"), cid.Undef); err != nil {
-				log.Warnw("writing error response", "error", err)
-				return
-			}
-			continue
-		}
-	}
-}
-
-func (h *NetworkStoreHandler) respondError(req uint64, uerr error, c cid.Cid) error {
-	var resp NetRpcResp
-	resp.ID = req
-	resp.Type = NRpcErr
-
-	nerr := NetRpcErr{
-		Type: NRpcErrGeneric,
-		Msg:  uerr.Error(),
-	}
-	if ipld.IsNotFound(uerr) {
-		nerr.Type = NRpcErrNotFound
-		nerr.Cid = &c
-	}
-
-	var edata bytes.Buffer
-	if err := nerr.MarshalCBOR(&edata); err != nil {
-		return xerrors.Errorf("marshaling error data: %w", err)
-	}
-
-	resp.Data = edata.Bytes()
-
-	var msg bytes.Buffer
-	if err := resp.MarshalCBOR(&msg); err != nil {
-		return xerrors.Errorf("marshaling error response: %w", err)
-	}
-
-	if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
-		return xerrors.Errorf("write error response: %w", err)
-	}
-
-	return nil
-}
-
-func (h *NetworkStoreHandler) respond(req uint64, rt NetRPCRespType, data []byte) error {
-	var resp NetRpcResp
-	resp.ID = req
-	resp.Type = rt
-	resp.Data = data
-
-	var msg bytes.Buffer
-	if err := resp.MarshalCBOR(&msg); err != nil {
-		return xerrors.Errorf("marshaling response: %w", err)
-	}
-
-	if err := h.msgStream.WriteMsg(msg.Bytes()); err != nil {
-		return xerrors.Errorf("write response: %w", err)
-	}
-
-	return nil
-}

blockstore/net_test.go

@@ -1,63 +0,0 @@
-package blockstore
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"testing"
-
-	block "github.com/ipfs/go-block-format"
-	ipld "github.com/ipfs/go-ipld-format"
-	"github.com/libp2p/go-msgio"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNetBstore(t *testing.T) {
-	ctx := context.Background()
-
-	cr, sw := io.Pipe()
-	sr, cw := io.Pipe()
-
-	cm := msgio.Combine(msgio.NewWriter(cw), msgio.NewReader(cr))
-	sm := msgio.Combine(msgio.NewWriter(sw), msgio.NewReader(sr))
-
-	bbs := NewMemorySync()
-	_ = HandleNetBstoreStream(ctx, bbs, sm)
-
-	nbs := NewNetworkStore(cm)
-
-	tb1 := block.NewBlock([]byte("aoeu"))
-
-	h, err := nbs.Has(ctx, tb1.Cid())
-	require.NoError(t, err)
-	require.False(t, h)
-
-	err = nbs.Put(ctx, tb1)
-	require.NoError(t, err)
-
-	h, err = nbs.Has(ctx, tb1.Cid())
-	require.NoError(t, err)
-	require.True(t, h)
-
-	sz, err := nbs.GetSize(ctx, tb1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, 4, sz)
-
-	err = nbs.DeleteBlock(ctx, tb1.Cid())
-	require.NoError(t, err)
-
-	h, err = nbs.Has(ctx, tb1.Cid())
-	require.NoError(t, err)
-	require.False(t, h)
-
-	_, err = nbs.Get(ctx, tb1.Cid())
-	fmt.Println(err)
-	require.True(t, ipld.IsNotFound(err))
-
-	err = nbs.Put(ctx, tb1)
-	require.NoError(t, err)
-
-	b, err := nbs.Get(ctx, tb1.Cid())
-	require.NoError(t, err)
-	require.Equal(t, "aoeu", string(b.RawData()))
-}

blockstore/net_ws.go

@@ -1,100 +0,0 @@
-package blockstore
-
-import (
-	"bytes"
-	"context"
-
-	"github.com/gorilla/websocket"
-	"github.com/libp2p/go-msgio"
-	"golang.org/x/xerrors"
-)
-
-type wsWrapper struct {
-	wc *websocket.Conn
-
-	nextMsg []byte
-}
-
-func (w *wsWrapper) Read(b []byte) (int, error) {
-	return 0, xerrors.New("read unsupported")
-}
-
-func (w *wsWrapper) ReadMsg() ([]byte, error) {
-	if w.nextMsg != nil {
-		nm := w.nextMsg
-		w.nextMsg = nil
-		return nm, nil
-	}
-
-	mt, r, err := w.wc.NextReader()
-	if err != nil {
-		return nil, err
-	}
-
-	switch mt {
-	case websocket.BinaryMessage, websocket.TextMessage:
-	default:
-		return nil, xerrors.Errorf("unexpected message type")
-	}
-
-	// todo pool
-	// todo limit sizes
-	var mbuf bytes.Buffer
-	if _, err := mbuf.ReadFrom(r); err != nil {
-		return nil, err
-	}
-
-	return mbuf.Bytes(), nil
-}
-
-func (w *wsWrapper) ReleaseMsg(bytes []byte) {
-	// todo use a pool
-}
-
-func (w *wsWrapper) NextMsgLen() (int, error) {
-	if w.nextMsg != nil {
-		return len(w.nextMsg), nil
-	}
-
-	mt, msg, err := w.wc.ReadMessage()
-	if err != nil {
-		return 0, err
-	}
-
-	switch mt {
-	case websocket.BinaryMessage, websocket.TextMessage:
-	default:
-		return 0, xerrors.Errorf("unexpected message type")
-	}
-
-	w.nextMsg = msg
-	return len(w.nextMsg), nil
-}
-
-func (w *wsWrapper) Write(bytes []byte) (int, error) {
-	return 0, xerrors.New("write unsupported")
-}
-
-func (w *wsWrapper) WriteMsg(bytes []byte) error {
-	return w.wc.WriteMessage(websocket.BinaryMessage, bytes)
-}
-
-func (w *wsWrapper) Close() error {
-	return w.wc.Close()
-}
-
-var _ msgio.ReadWriteCloser = &wsWrapper{}
-
-func wsConnToMio(wc *websocket.Conn) msgio.ReadWriteCloser {
-	return &wsWrapper{
-		wc: wc,
-	}
-}
-
-func HandleNetBstoreWS(ctx context.Context, bs Blockstore, wc *websocket.Conn) *NetworkStoreHandler {
-	return HandleNetBstoreStream(ctx, bs, wsConnToMio(wc))
-}
-
-func NewNetworkStoreWS(wc *websocket.Conn) *NetworkStore {
-	return NewNetworkStore(wsConnToMio(wc))
-}

blockstore/splitstore/checkpoint.go

@@ -5,9 +5,10 @@ import (
 	"io"
 	"os"
 
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
 )
 
 type Checkpoint struct {

blockstore/splitstore/coldset.go

@@ -5,8 +5,9 @@ import (
 	"io"
 	"os"
 
-	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
 )
 
 type ColdSetWriter struct {

blockstore/splitstore/debug.go

@@ -12,10 +12,11 @@ import (
 	"sync"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
 )
 
 type debugLog struct {

blockstore/splitstore/markset.go

@@ -3,8 +3,9 @@ package splitstore
 import (
 	"errors"
 
-	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
 )
 
 var errMarkSetClosed = errors.New("markset closed")

blockstore/splitstore/markset_badger.go

@@ -6,13 +6,13 @@ import (
 	"runtime"
 	"sync"
 
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
-	"github.com/ipfs/go-cid"
-	"go.uber.org/zap"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/system"
+	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v2/options"
+	"go.uber.org/zap"
+
+	cid "github.com/ipfs/go-cid"
 )
 
 type BadgerMarkSetEnv struct {
@@ -351,7 +351,7 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 	persist := s.persist
 	s.mx.RUnlock()
 
-	if persist && !system.BadgerFsyncDisable { // WARNING: disabling sync makes recovery from crash during critical section unsound
+	if persist {
 		return s.db.Sync()
 	}
 

blockstore/splitstore/markset_map.go

@@ -7,8 +7,9 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
 )
 
 type MapMarkSetEnv struct {

blockstore/splitstore/markset_test.go

@@ -1,10 +1,10 @@
-// stm: #unit
+//stm: #unit
 package splitstore
 
 import (
 	"testing"
 
-	"github.com/ipfs/go-cid"
+	cid "github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 )
 

blockstore/splitstore/splitstore.go

@@ -8,22 +8,22 @@ import (
 	"sync/atomic"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	dstore "github.com/ipfs/go-datastore"
-	ipld "github.com/ipfs/go-ipld-format"
-	logging "github.com/ipfs/go-log/v2"
-	"go.opencensus.io/stats"
 	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	dstore "github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
 
+	"github.com/filecoin-project/go-state-types/abi"
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
+
+	"go.opencensus.io/stats"
 )
 
 var (
@@ -43,16 +43,8 @@ var (
 	// compactionIndexKey stores the compaction index (serial number)
 	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
 
-	// stores the prune index (serial number)
-	pruneIndexKey = dstore.NewKey("/splitstore/pruneIndex")
-
-	// stores the base epoch of last prune in the metadata store
-	pruneEpochKey = dstore.NewKey("/splitstore/pruneEpoch")
-
 	log = logging.Logger("splitstore")
 
-	errClosing = errors.New("splitstore is closing")
-
 	// set this to true if you are debugging the splitstore to enable debug logging
 	enableDebugLog = false
 	// set this to true if you want to track origin stack traces in the write log
@@ -62,16 +54,6 @@ var (
 	upgradeBoundary = build.Finality
 )
 
-type CompactType int
-
-const (
-	none CompactType = iota
-	warmup
-	hot
-	cold
-	check
-)
-
 func init() {
 	if os.Getenv("LOTUS_SPLITSTORE_DEBUG_LOG") == "1" {
 		enableDebugLog = true
@@ -98,10 +80,6 @@ type Config struct {
 	// and directly purges cold blocks.
 	DiscardColdBlocks bool
 
-	// UniversalColdBlocks indicates whether all blocks being garbage collected and purged
-	// from the hotstore should be written to the cold store
-	UniversalColdBlocks bool
-
 	// HotstoreMessageRetention indicates the hotstore retention policy for messages.
 	// It has the following semantics:
 	// - a value of 0 will only retain messages within the compaction boundary (4 finalities)
@@ -115,23 +93,6 @@ type Config struct {
 	// A positive value is the number of compactions before a full GC is performed;
 	// a value of 1 will perform full GC in every compaction.
 	HotStoreFullGCFrequency uint64
-
-	// HotstoreMaxSpaceTarget suggests the max allowed space the hotstore can take.
-	// This is not a hard limit, it is possible for the hotstore to exceed the target
-	// for example if state grows massively between compactions. The splitstore
-	// will make a best effort to avoid overflowing the target and in practice should
-	// never overflow.  This field is used when doing GC at the end of a compaction to
-	// adaptively choose moving GC
-	HotstoreMaxSpaceTarget uint64
-
-	// Moving GC will be triggered when total moving size exceeds
-	// HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold
-	HotstoreMaxSpaceThreshold uint64
-
-	// Safety buffer to prevent moving GC from overflowing disk.
-	// Moving GC will not occur when total moving size exceeds
-	// HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer
-	HotstoreMaxSpaceSafetyBuffer uint64
 }
 
 // ChainAccessor allows the Splitstore to access the chain. It will most likely
@@ -156,17 +117,15 @@ type hotstore interface {
 }
 
 type SplitStore struct {
-	compacting  int32       // flag for when compaction is in progress
+	compacting int32 // compaction/prune/warmup in progress
-	compactType CompactType // compaction type, protected by compacting atomic, only meaningful when compacting == 1
+	closing    int32 // the splitstore is closing
-	closing     int32       // the splitstore is closing
 
 	cfg  *Config
 	path string
 
 	mx          sync.Mutex
-	warmupEpoch atomic.Int64
+	warmupEpoch abi.ChainEpoch // protected by mx
 	baseEpoch   abi.ChainEpoch // protected by compaction lock
-	pruneEpoch  abi.ChainEpoch // protected by compaction lock
 
 	headChangeMx sync.Mutex
 
@@ -181,16 +140,10 @@ type SplitStore struct {
 	markSetSize int64
 
 	compactionIndex int64
-	pruneIndex      int64
 
 	ctx    context.Context
 	cancel func()
 
-	outOfSync         int32 // for fast checking
-	chainSyncMx       sync.Mutex
-	chainSyncCond     sync.Cond
-	chainSyncFinished bool // protected by chainSyncMx
-
 	debug *debugLog
 
 	// transactional protection for concurrent read/writes during compaction
@@ -217,17 +170,6 @@ type SplitStore struct {
 
 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
-
-	// dag sizes measured during latest compaction
-	// logged and used for GC strategy
-
-	// protected by compaction lock
-	szWalk          int64
-	szProtectedTxns int64
-	szKeys          int64 // approximate, not counting keys protected when entering critical section
-
-	// protected by txnLk
-	szMarkedLiveRefs int64
 }
 
 var _ bstore.Blockstore = (*SplitStore)(nil)
@@ -265,7 +207,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 
 	ss.txnViewsCond.L = &ss.txnViewsMx
 	ss.txnSyncCond.L = &ss.txnSyncMx
-	ss.chainSyncCond.L = &ss.chainSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())
 
 	ss.reifyCond.L = &ss.reifyMx
@@ -286,13 +227,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 			return nil, xerrors.Errorf("error resuming compaction: %w", err)
 		}
 	}
-	if ss.pruneCheckpointExists() {
-		log.Info("found prune checkpoint; resuming prune")
-		if err := ss.completePrune(); err != nil {
-			markSetEnv.Close() //nolint:errcheck
-			return nil, xerrors.Errorf("error resuming prune: %w", err)
-		}
-	}
 
 	return ss, nil
 }
@@ -326,14 +260,8 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		if has {
 			return s.has(cid)
 		}
-		switch s.compactType {
+
-		case hot:
+		return s.cold.Has(ctx, cid)
-			return s.cold.Has(ctx, cid)
-		case cold:
-			return s.hot.Has(ctx, cid)
-		default:
-			return false, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
-		}
 	}
 
 	has, err := s.hot.Has(ctx, cid)
@@ -348,11 +276,8 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	}
 
 	has, err = s.cold.Has(ctx, cid)
-	if has {
+	if has && bstore.IsHotView(ctx) {
-		s.trackTxnRef(cid)
+		s.reifyColdObject(cid)
-		if bstore.IsHotView(ctx) {
-			s.reifyColdObject(cid)
-		}
 	}
 
 	return has, err
@@ -382,31 +307,24 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 		if has {
 			return s.get(cid)
 		}
-		switch s.compactType {
+
-		case hot:
+		return s.cold.Get(ctx, cid)
-			return s.cold.Get(ctx, cid)
-		case cold:
-			return s.hot.Get(ctx, cid)
-		default:
-			return nil, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
-		}
 	}
 
 	blk, err := s.hot.Get(ctx, cid)
 
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		s.trackTxnRef(cid)
 		return blk, nil
 
-	case ipld.IsNotFound(err):
+	case bstore.ErrNotFound:
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -443,31 +361,24 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 		if has {
 			return s.getSize(cid)
 		}
-		switch s.compactType {
+
-		case hot:
+		return s.cold.GetSize(ctx, cid)
-			return s.cold.GetSize(ctx, cid)
-		case cold:
-			return s.hot.GetSize(ctx, cid)
-		default:
-			return 0, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
-		}
 	}
 
 	size, err := s.hot.GetSize(ctx, cid)
 
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		s.trackTxnRef(cid)
 		return size, nil
 
-	case ipld.IsNotFound(err):
+	case bstore.ErrNotFound:
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
 
 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
-			s.trackTxnRef(cid)
 			if bstore.IsHotView(ctx) {
 				s.reifyColdObject(cid)
 			}
@@ -481,23 +392,6 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	}
 }
 
-func (s *SplitStore) Flush(ctx context.Context) error {
-	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()
-
-	if err := s.cold.Flush(ctx); err != nil {
-		return err
-	}
-	if err := s.hot.Flush(ctx); err != nil {
-		return err
-	}
-	if err := s.ds.Sync(ctx, dstore.Key{}); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	if isIdentiyCid(blk.Cid()) {
 		return nil
@@ -514,12 +408,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 	s.debug.LogWrite(blk)
 
 	// critical section
-	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
+	if s.txnMarkSet != nil {
 		s.markLiveRefs([]cid.Cid{blk.Cid()})
 		return nil
 	}
-	s.trackTxnRef(blk.Cid())
 
+	s.trackTxnRef(blk.Cid())
 	return nil
 }
 
@@ -565,12 +459,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 	s.debug.LogWriteMany(blks)
 
 	// critical section
-	if s.txnMarkSet != nil && s.compactType == hot { // puts only touch hot store
+	if s.txnMarkSet != nil {
 		s.markLiveRefs(batch)
 		return nil
 	}
-	s.trackTxnRefMany(batch)
 
+	s.trackTxnRefMany(batch)
 	return nil
 }
 
@@ -642,14 +536,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		if has {
 			return s.view(cid, cb)
 		}
-		switch s.compactType {
+
-		case hot:
+		return s.cold.View(ctx, cid, cb)
-			return s.cold.View(ctx, cid, cb)
-		case cold:
-			return s.hot.View(ctx, cid, cb)
-		default:
-			return xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
-		}
 	}
 
 	// views are (optimistically) protected two-fold:
@@ -664,7 +552,8 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 	defer s.viewDone()
 
 	err := s.hot.View(ctx, cid, cb)
-	if ipld.IsNotFound(err) {
+	switch err {
+	case bstore.ErrNotFound:
 		if s.isWarm() {
 			s.debug.LogReadMiss(cid)
 		}
@@ -678,12 +567,16 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
+
+	default:
+		return err
 	}
-	return err
 }
 
 func (s *SplitStore) isWarm() bool {
-	return s.warmupEpoch.Load() > 0
+	s.mx.Lock()
+	defer s.mx.Unlock()
+	return s.warmupEpoch > 0
 }
 
 // State tracking
@@ -733,28 +626,11 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		return xerrors.Errorf("error loading base epoch: %w", err)
 	}
 
-	// load prune epoch from metadata ds
-	bs, err = s.ds.Get(s.ctx, pruneEpochKey)
-	switch err {
-	case nil:
-		s.pruneEpoch = bytesToEpoch(bs)
-	case dstore.ErrNotFound:
-		if curTs == nil {
-			//this can happen in some tests
-			break
-		}
-		if err := s.setPruneEpoch(curTs.Height()); err != nil {
-			return xerrors.Errorf("error saving prune epoch: %w", err)
-		}
-	default:
-		return xerrors.Errorf("error loading prune epoch: %w", err)
-	}
-
 	// load warmup epoch from metadata ds
 	bs, err = s.ds.Get(s.ctx, warmupEpochKey)
 	switch err {
 	case nil:
-		s.warmupEpoch.Store(bytesToInt64(bs))
+		s.warmupEpoch = bytesToEpoch(bs)
 
 	case dstore.ErrNotFound:
 		warmup = true
@@ -788,7 +664,7 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		return xerrors.Errorf("error loading compaction index: %w", err)
 	}
 
-	log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch.Load())
+	log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch)
 
 	if warmup {
 		err = s.warmup(curTs)
@@ -825,11 +701,6 @@ func (s *SplitStore) Close() error {
 		s.txnSyncCond.Broadcast()
 		s.txnSyncMx.Unlock()
 
-		s.chainSyncMx.Lock()
-		s.chainSyncFinished = true
-		s.chainSyncCond.Broadcast()
-		s.chainSyncMx.Unlock()
-
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)
@@ -854,8 +725,3 @@ func (s *SplitStore) setBaseEpoch(epoch abi.ChainEpoch) error {
 	s.baseEpoch = epoch
 	return s.ds.Put(s.ctx, baseEpochKey, epochToBytes(epoch))
 }
-
-func (s *SplitStore) setPruneEpoch(epoch abi.ChainEpoch) error {
-	s.pruneEpoch = epoch
-	return s.ds.Put(s.ctx, pruneEpochKey, epochToBytes(epoch))
-}

blockstore/splitstore/splitstore_check.go

@@ -8,9 +8,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
+	cid "github.com/ipfs/go-cid"
+
 	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -25,7 +26,6 @@ func (s *SplitStore) Check() error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("can't acquire compaction lock; compacting operation in progress")
 	}
-	s.compactType = check
 
 	if s.compactionIndex == 0 {
 		atomic.StoreInt32(&s.compacting, 0)
@@ -95,7 +95,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	}
 	defer visitor.Close() //nolint
 
-	size := s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
+	err = s.walkChain(curTs, boundaryEpoch, boundaryEpoch, visitor,
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
 				return errStopWalk
@@ -125,7 +125,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 			}
 
 			return nil
-		}, func(cid.Cid) error { return nil })
+		})
 
 	if err != nil {
 		err = xerrors.Errorf("error walking chain: %w", err)
@@ -133,7 +133,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 		return err
 	}
 
-	log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt, "walk size", size)
+	log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
 	write("--")
 	write("cold: %d missing: %d", *coldCnt, *missingCnt)
 	write("DONE")
@@ -145,10 +145,8 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 func (s *SplitStore) Info() map[string]interface{} {
 	info := make(map[string]interface{})
 	info["base epoch"] = s.baseEpoch
-	info["warmup epoch"] = s.warmupEpoch.Load()
+	info["warmup epoch"] = s.warmupEpoch
 	info["compactions"] = s.compactionIndex
-	info["prunes"] = s.pruneIndex
-	info["compacting"] = s.compacting == 1
 
 	sizer, ok := s.hot.(bstore.BlockstoreSize)
 	if ok {

blockstore/splitstore/splitstore_compact.go

@@ -10,19 +10,20 @@ import (
 	"sync/atomic"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	"go.opencensus.io/stats"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
+
+	"go.opencensus.io/stats"
 )
 
 var (
@@ -52,12 +53,6 @@ var (
 	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
 	// we have synced.
 	SyncWaitTime = 30 * time.Second
-
-	// This is a testing flag that should always be true when running a node. itests rely on the rough hack
-	// of starting genesis so far in the past that they exercise catchup mining to mine
-	// blocks quickly and so disabling syncgap checking is necessary to test compaction
-	// without a deep structural improvement of itests.
-	CheckSyncGap = true
 )
 
 var (
@@ -66,9 +61,7 @@ var (
 )
 
 const (
-	batchSize              = 16384
+	batchSize = 16384
-	cidKeySize             = 128
-	purgeWorkSliceDuration = time.Second
 )
 
 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -89,38 +82,10 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	//       this is guaranteed by the chainstore, and it is pervasive in all lotus
 	//       -- if that ever changes then all hell will break loose in general and
 	//       we will have a rance to protectTipSets here.
-	//      Regardless, we put a mutex in HeadChange just to be safe
+	//      Reagrdless, we put a mutex in HeadChange just to be safe
 
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
-		// we are currently compacting
+		// we are currently compacting -- protect the new tipset(s)
-		// 1. Signal sync condition to yield compaction when out of sync and resume when in sync
-		timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
-		if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
-			/* Chain out of sync */
-			if atomic.CompareAndSwapInt32(&s.outOfSync, 0, 1) {
-				// transition from in sync to out of sync
-				s.chainSyncMx.Lock()
-				s.chainSyncFinished = false
-				s.chainSyncMx.Unlock()
-			}
-			// already out of sync, no signaling necessary
-
-		}
-		// TODO: ok to use hysteresis with no transitions between 30s and 1m?
-		if time.Since(timestamp) < SyncWaitTime {
-			/* Chain in sync */
-			if atomic.CompareAndSwapInt32(&s.outOfSync, 0, 0) {
-				// already in sync, no signaling necessary
-			} else {
-				// transition from out of sync to in sync
-				s.chainSyncMx.Lock()
-				s.chainSyncFinished = true
-				s.chainSyncCond.Broadcast()
-				s.chainSyncMx.Unlock()
-			}
-
-		}
-		// 2. protect the new tipset(s)
 		s.protectTipSets(apply)
 		return nil
 	}
@@ -132,8 +97,7 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	}
 
 	timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
-
+	if time.Since(timestamp) > SyncGapTime {
-	if CheckSyncGap && time.Since(timestamp) > SyncGapTime {
 		// don't attempt compaction before we have caught up syncing
 		atomic.StoreInt32(&s.compacting, 0)
 		return nil
@@ -148,7 +112,6 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 	if epoch-s.baseEpoch > CompactionThreshold {
 		// it's time to compact -- prepare the transaction and go!
 		s.beginTxnProtect()
-		s.compactType = hot
 		go func() {
 			defer atomic.StoreInt32(&s.compacting, 0)
 			defer s.endTxnProtect()
@@ -160,7 +123,6 @@ func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
 
 			log.Infow("compaction done", "took", time.Since(start))
 		}()
-		// only prune if auto prune is enabled and after at least one compaction
 	} else {
 		// no compaction necessary
 		atomic.StoreInt32(&s.compacting, 0)
@@ -204,8 +166,6 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 		timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
 		doSync := time.Since(timestamp) < SyncWaitTime
 		go func() {
-			// we are holding the txnLk while marking
-			// so critical section cannot delete
 			if doSync {
 				defer func() {
 					s.txnSyncMx.Lock()
@@ -229,11 +189,9 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
 	log.Debugf("marking %d live refs", len(cids))
 	startMark := time.Now()
 
-	szMarked := new(int64)
-
 	count := new(int32)
 	visitor := newConcurrentVisitor()
-	walkObject := func(c cid.Cid) (int64, error) {
+	walkObject := func(c cid.Cid) error {
 		return s.walkObjectIncomplete(c, visitor,
 			func(c cid.Cid) error {
 				if isUnitaryObject(c) {
@@ -260,12 +218,10 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
 
 	// optimize the common case of single put
 	if len(cids) == 1 {
-		sz, err := walkObject(cids[0])
+		if err := walkObject(cids[0]); err != nil {
-		if err != nil {
 			log.Errorf("error marking tipset refs: %s", err)
 		}
 		log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
-		atomic.AddInt64(szMarked, sz)
 		return
 	}
 
@@ -277,11 +233,9 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
 
 	worker := func() error {
 		for c := range workch {
-			sz, err := walkObject(c)
+			if err := walkObject(c); err != nil {
-			if err != nil {
 				return err
 			}
-			atomic.AddInt64(szMarked, sz)
 		}
 
 		return nil
@@ -304,8 +258,7 @@ func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
 		log.Errorf("error marking tipset refs: %s", err)
 	}
 
-	log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count, "size marked", *szMarked)
+	log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
-	s.szMarkedLiveRefs += atomic.LoadInt64(szMarked)
 }
 
 // transactionally protect a view
@@ -398,7 +351,6 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
 
 		log.Infow("protecting transactional references", "refs", len(txnRefs))
 		count := 0
-		sz := new(int64)
 		workch := make(chan cid.Cid, len(txnRefs))
 		startProtect := time.Now()
 
@@ -431,11 +383,10 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
 
 		worker := func() error {
 			for c := range workch {
-				szTxn, err := s.doTxnProtect(c, markSet)
+				err := s.doTxnProtect(c, markSet)
 				if err != nil {
 					return xerrors.Errorf("error protecting transactional references to %s: %w", c, err)
 				}
-				atomic.AddInt64(sz, szTxn)
 			}
 			return nil
 		}
@@ -448,16 +399,16 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
 		if err := g.Wait(); err != nil {
 			return err
 		}
-		s.szProtectedTxns += atomic.LoadInt64(sz)
+
-		log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count, "protected size", sz)
+		log.Infow("protecting transactional refs done", "took", time.Since(startProtect), "protected", count)
 	}
 }
 
 // transactionally protect a reference by walking the object and marking.
 // concurrent markings are short circuited by checking the markset.
-func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error) {
+func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
 	if err := s.checkClosing(); err != nil {
-		return 0, err
+		return err
 	}
 
 	// Note: cold objects are deleted heaviest first, so the consituents of an object
@@ -481,7 +432,7 @@ func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) (int64, error)
 		},
 		func(c cid.Cid) error {
 			if s.txnMissing != nil {
-				log.Debugf("missing object reference %s in %s", c, root)
+				log.Warnf("missing object reference %s in %s", c, root)
 				s.txnRefsMx.Lock()
 				s.txnMissing[c] = struct{}{}
 				s.txnRefsMx.Unlock()
@@ -524,7 +475,6 @@ func (s *SplitStore) applyProtectors() error {
 // - At this point we are ready to begin purging:
 //   - We sort cold objects heaviest first, so as to never delete the consituents of a DAG before the DAG itself (which would leave dangling references)
 //   - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
-//
 // - We then end the transaction and compact/gc the hotstore.
 func (s *SplitStore) compact(curTs *types.TipSet) {
 	log.Info("waiting for active views to complete")
@@ -548,7 +498,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		// might be potentially inconsistent; abort compaction and notify the user to intervene.
 		return xerrors.Errorf("checkpoint exists; aborting compaction")
 	}
-	s.clearSizeMeasurements()
 
 	currentEpoch := curTs.Height()
 	boundaryEpoch := currentEpoch - CompactionBoundary
@@ -568,13 +517,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	defer markSet.Close() //nolint:errcheck
 	defer s.debug.Flush()
 
-	coldSet, err := s.markSetEnv.New("cold", s.markSetSize)
+	if err := s.checkClosing(); err != nil {
-	if err != nil {
-		return xerrors.Errorf("error creating cold mark set: %w", err)
-	}
-	defer coldSet.Close() //nolint:errcheck
-
-	if err := s.checkYield(); err != nil {
 		return err
 	}
 
@@ -592,52 +535,25 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	startMark := time.Now()
 
 	count := new(int64)
+	err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
 
-	coldCount := new(int64)
+			visit, err := markSet.Visit(c)
-	fCold := func(c cid.Cid) error {
+			if err != nil {
-		// Writes to cold set optimized away in universal and discard mode
+				return xerrors.Errorf("error visiting object: %w", err)
-		//
+			}
-		// Nothing gets written to cold store in discard mode so no cold objects to write
+
-		// Everything not marked hot gets written to cold store in universal mode so no need to track cold objects separately
+			if !visit {
-		if s.cfg.DiscardColdBlocks || s.cfg.UniversalColdBlocks {
+				return errStopWalk
+			}
+
+			atomic.AddInt64(count, 1)
 			return nil
-		}
+		})
 
-		if isUnitaryObject(c) {
-			return errStopWalk
-		}
-
-		visit, err := coldSet.Visit(c)
-		if err != nil {
-			return xerrors.Errorf("error visiting object: %w", err)
-		}
-
-		if !visit {
-			return errStopWalk
-		}
-
-		atomic.AddInt64(coldCount, 1)
-		return nil
-	}
-	fHot := func(c cid.Cid) error {
-		if isUnitaryObject(c) {
-			return errStopWalk
-		}
-
-		visit, err := markSet.Visit(c)
-		if err != nil {
-			return xerrors.Errorf("error visiting object: %w", err)
-		}
-
-		if !visit {
-			return errStopWalk
-		}
-
-		atomic.AddInt64(count, 1)
-		return nil
-	}
-
-	err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{}, fHot, fCold)
 	if err != nil {
 		return xerrors.Errorf("error marking: %w", err)
 	}
@@ -646,7 +562,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 	log.Infow("marking done", "took", time.Since(startMark), "marked", *count)
 
-	if err := s.checkYield(); err != nil {
+	if err := s.checkClosing(); err != nil {
 		return err
 	}
 
@@ -656,7 +572,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error protecting transactional refs: %w", err)
 	}
 
-	if err := s.checkYield(); err != nil {
+	if err := s.checkClosing(); err != nil {
 		return err
 	}
 
@@ -670,14 +586,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	}
 	defer coldw.Close() //nolint:errcheck
 
-	purgew, err := NewColdSetWriter(s.discardSetPath())
-	if err != nil {
-		return xerrors.Errorf("error creating deadset: %w", err)
-	}
-	defer purgew.Close() //nolint:errcheck
-
 	// some stats for logging
-	var hotCnt, coldCnt, purgeCnt int64
+	var hotCnt, coldCnt int
+
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -690,50 +601,30 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 			return nil
 		}
 
-		// it needs to be removed from hot store, mark it as candidate for purge
+		// it's cold, mark it as candidate for move
-		if err := purgew.Write(c); err != nil {
-			return xerrors.Errorf("error writing cid to purge set: %w", err)
-		}
-		purgeCnt++
-
-		coldMark, err := coldSet.Has(c)
-		if err != nil {
-			return xerrors.Errorf("error checking cold mark set for %s: %w", c, err)
-		}
-
-		// Discard mode: coldMark == false, s.cfg.UniversalColdBlocks == false, always return here, no writes to cold store
-		// Universal mode: coldMark == false, s.cfg.UniversalColdBlocks == true, never stop here, all writes to cold store
-		// Otherwise: s.cfg.UniversalColdBlocks == false, if !coldMark stop here and don't write to cold store, if coldMark continue and write to cold store
-		if !coldMark && !s.cfg.UniversalColdBlocks { // universal mode means mark everything as cold
-			return nil
-		}
-
-		// it's cold, mark as candidate for move
 		if err := coldw.Write(c); err != nil {
-			return xerrors.Errorf("error writing cid to cold set")
+			return xerrors.Errorf("error writing cid to coldstore: %w", err)
 		}
 		coldCnt++
 
 		return nil
 	})
+
 	if err != nil {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}
-	if err := purgew.Close(); err != nil {
+
-		return xerrors.Errorf("erroring closing purgeset: %w", err)
-	}
 	if err := coldw.Close(); err != nil {
 		return xerrors.Errorf("error closing coldset: %w", err)
 	}
 
 	log.Infow("cold collection done", "took", time.Since(startCollect))
 
-	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "purge", purgeCnt)
+	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
-	s.szKeys = hotCnt * cidKeySize
+	stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
-	stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(hotCnt))
+	stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
-	stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(coldCnt))
 
-	if err := s.checkYield(); err != nil {
+	if err := s.checkClosing(); err != nil {
 		return err
 	}
 
@@ -742,7 +633,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	// possibly delete objects we didn't have when we were collecting cold objects)
 	s.waitForMissingRefs(markSet)
 
-	if err := s.checkYield(); err != nil {
+	if err := s.checkClosing(); err != nil {
 		return err
 	}
 
@@ -762,7 +653,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		}
 		log.Infow("moving done", "took", time.Since(startMove))
 
-		if err := s.checkYield(); err != nil {
+		if err := s.checkClosing(); err != nil {
 			return err
 		}
 
@@ -771,12 +662,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		}
 	}
 
-	purger, err := NewColdSetReader(s.discardSetPath())
-	if err != nil {
-		return xerrors.Errorf("error opening coldset: %w", err)
-	}
-	defer purger.Close() //nolint:errcheck
-
 	// 4. Purge cold objects with checkpointing for recovery.
 	// This is the critical section of compaction, whereby any cold object not in the markSet is
 	// considered already deleted.
@@ -793,7 +678,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	}
 
 	// wait for the head to catch up so that the current tipset is marked
-	s.waitForTxnSync()
+	s.waitForSync()
 
 	if err := s.checkClosing(); err != nil {
 		return err
@@ -808,13 +693,13 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	// 5. purge cold objects from the hotstore, taking protected references into account
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
-	err = s.purge(purger, checkpoint, markSet)
+	err = s.purge(coldr, checkpoint, markSet)
 	if err != nil {
 		return xerrors.Errorf("error purging cold objects: %w", err)
 	}
 	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
+
 	s.endCriticalSection()
-	log.Infow("critical section done", "total protected size", s.szProtectedTxns, "total marked live size", s.szMarkedLiveRefs)
 
 	if err := checkpoint.Close(); err != nil {
 		log.Warnf("error closing checkpoint: %s", err)
@@ -828,13 +713,10 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	if err := os.Remove(s.coldSetPath()); err != nil {
 		log.Warnf("error removing coldset: %s", err)
 	}
-	if err := os.Remove(s.discardSetPath()); err != nil {
-		log.Warnf("error removing discardset: %s", err)
-	}
 
 	// we are done; do some housekeeping
 	s.endTxnProtect()
-	s.gcHotAfterCompaction()
+	s.gcHotstore()
 
 	err = s.setBaseEpoch(boundaryEpoch)
 	if err != nil {
@@ -894,12 +776,8 @@ func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
 	return nil
 }
 
-func (s *SplitStore) waitForTxnSync() {
+func (s *SplitStore) waitForSync() {
 	log.Info("waiting for sync")
-	if !CheckSyncGap {
-		log.Warnf("If you see this outside of test it is a serious splitstore issue")
-		return
-	}
 	startWait := time.Now()
 	defer func() {
 		log.Infow("waiting for sync done", "took", time.Since(startWait))
@@ -913,25 +791,6 @@ func (s *SplitStore) waitForTxnSync() {
 	}
 }
 
-// Block compaction operations if chain sync has fallen behind
-func (s *SplitStore) waitForSync() {
-	if atomic.LoadInt32(&s.outOfSync) == 0 {
-		return
-	}
-	s.chainSyncMx.Lock()
-	defer s.chainSyncMx.Unlock()
-
-	for !s.chainSyncFinished {
-		s.chainSyncCond.Wait()
-	}
-}
-
-// Combined sync and closing check
-func (s *SplitStore) checkYield() error {
-	s.waitForSync()
-	return s.checkClosing()
-}
-
 func (s *SplitStore) endTxnProtect() {
 	s.txnLk.Lock()
 	defer s.txnLk.Unlock()
@@ -958,7 +817,7 @@ func (s *SplitStore) endCriticalSection() {
 }
 
 func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
-	visitor ObjectVisitor, fHot, fCold func(cid.Cid) error) error {
+	visitor ObjectVisitor, f func(cid.Cid) error) error {
 	var walked ObjectVisitor
 	var mx sync.Mutex
 	// we copy the tipset first into a new slice, which allows us to reuse it in every epoch.
@@ -966,11 +825,6 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 	copy(toWalk, ts.Cids())
 	walkCnt := new(int64)
 	scanCnt := new(int64)
-	szWalk := new(int64)
-
-	tsRef := func(blkCids []cid.Cid) (cid.Cid, error) {
-		return types.NewTipSetKey(blkCids...).Cid()
-	}
 
 	stopWalk := func(_ cid.Cid) error { return errStopWalk }
 
@@ -985,7 +839,7 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 
 		atomic.AddInt64(walkCnt, 1)
 
-		if err := fHot(c); err != nil {
+		if err := f(c); err != nil {
 			return err
 		}
 
@@ -993,73 +847,39 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 		err = s.view(c, func(data []byte) error {
 			return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
 		})
+
 		if err != nil {
 			return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
 		}
 
-		// tipset CID references are retained
-		pRef, err := tsRef(hdr.Parents)
-		if err != nil {
-			return xerrors.Errorf("error computing cid reference to parent tipset")
-		}
-		sz, err := s.walkObjectIncomplete(pRef, visitor, fHot, stopWalk)
-		if err != nil {
-			return xerrors.Errorf("error walking parent tipset cid reference")
-		}
-		atomic.AddInt64(szWalk, sz)
-
 		// message are retained if within the inclMsgs boundary
 		if hdr.Height >= inclMsgs && hdr.Height > 0 {
 			if inclMsgs < inclState {
 				// we need to use walkObjectIncomplete here, as messages/receipts may be missing early on if we
 				// synced from snapshot and have a long HotStoreMessageRetentionPolicy.
-				sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fHot, stopWalk)
+				if err := s.walkObjectIncomplete(hdr.Messages, visitor, f, stopWalk); err != nil {
-				if err != nil {
 					return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
 				}
-				atomic.AddInt64(szWalk, sz)
 
-				sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk)
+				if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, f, stopWalk); err != nil {
-				if err != nil {
 					return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
 				}
-				atomic.AddInt64(szWalk, sz)
 			} else {
-				sz, err = s.walkObject(hdr.Messages, visitor, fHot)
+				if err := s.walkObject(hdr.Messages, visitor, f); err != nil {
-				if err != nil {
 					return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
 				}
-				atomic.AddInt64(szWalk, sz)
 
-				sz, err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fHot, stopWalk)
+				if err := s.walkObject(hdr.ParentMessageReceipts, visitor, f); err != nil {
-				if err != nil {
 					return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
 				}
-				atomic.AddInt64(szWalk, sz)
 			}
 		}
 
-		// messages and receipts outside of inclMsgs are included in the cold store
-		if hdr.Height < inclMsgs && hdr.Height > 0 {
-			sz, err := s.walkObjectIncomplete(hdr.Messages, visitor, fCold, stopWalk)
-			if err != nil {
-				return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
-			}
-			atomic.AddInt64(szWalk, sz)
-			sz, err = s.walkObjectIncomplete(hdr.ParentMessageReceipts, visitor, fCold, stopWalk)
-			if err != nil {
-				return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
-			}
-			atomic.AddInt64(szWalk, sz)
-		}
-
 		// state is only retained if within the inclState boundary, with the exception of genesis
 		if hdr.Height >= inclState || hdr.Height == 0 {
-			sz, err := s.walkObject(hdr.ParentStateRoot, visitor, fHot)
+			if err := s.walkObject(hdr.ParentStateRoot, visitor, f); err != nil {
-			if err != nil {
 				return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
 			}
-			atomic.AddInt64(szWalk, sz)
 			atomic.AddInt64(scanCnt, 1)
 		}
 
@@ -1072,20 +892,9 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 		return nil
 	}
 
-	// retain ref to chain head
-	hRef, err := tsRef(ts.Cids())
-	if err != nil {
-		return xerrors.Errorf("error computing cid reference to parent tipset")
-	}
-	sz, err := s.walkObjectIncomplete(hRef, visitor, fHot, stopWalk)
-	if err != nil {
-		return xerrors.Errorf("error walking parent tipset cid reference")
-	}
-	atomic.AddInt64(szWalk, sz)
-
 	for len(toWalk) > 0 {
 		// walking can take a while, so check this with every opportunity
-		if err := s.checkYield(); err != nil {
+		if err := s.checkClosing(); err != nil {
 			return err
 		}
 
@@ -1115,143 +924,133 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 					if err := walkBlock(c); err != nil {
 						return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
 					}
-
-					if err := s.checkYield(); err != nil {
-						return xerrors.Errorf("check yield: %w", err)
-					}
 				}
 				return nil
 			})
 		}
 
 		if err := g.Wait(); err != nil {
-			return xerrors.Errorf("walkBlock workers errored: %w", err)
+			return err
 		}
 	}
 
-	log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt, "walk size", szWalk)
+	log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt)
-	s.szWalk = atomic.LoadInt64(szWalk)
+
 	return nil
 }
 
-func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) (int64, error) {
+func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid) error) error {
-	var sz int64
 	visit, err := visitor.Visit(c)
 	if err != nil {
-		return 0, xerrors.Errorf("error visiting object: %w", err)
+		return xerrors.Errorf("error visiting object: %w", err)
 	}
 
 	if !visit {
-		return sz, nil
+		return nil
 	}
 
 	if err := f(c); err != nil {
 		if err == errStopWalk {
-			return sz, nil
+			return nil
 		}
 
-		return 0, err
+		return err
 	}
 
 	if c.Prefix().Codec != cid.DagCBOR {
-		return sz, nil
+		return nil
 	}
 
 	// check this before recursing
 	if err := s.checkClosing(); err != nil {
-		return 0, xerrors.Errorf("check closing: %w", err)
+		return err
 	}
 
 	var links []cid.Cid
 	err = s.view(c, func(data []byte) error {
-		sz += int64(len(data))
 		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
 			links = append(links, c)
 		})
 	})
 
 	if err != nil {
-		return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
+		return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
 	}
 
 	for _, c := range links {
-		szLink, err := s.walkObject(c, visitor, f)
+		err := s.walkObject(c, visitor, f)
 		if err != nil {
-			return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
+			return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
 		}
-		sz += szLink
 	}
 
-	return sz, nil
+	return nil
 }
 
 // like walkObject, but the object may be potentially incomplete (references missing)
-func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) (int64, error) {
+func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, missing func(cid.Cid) error) error {
-	var sz int64
 	visit, err := visitor.Visit(c)
 	if err != nil {
-		return 0, xerrors.Errorf("error visiting object: %w", err)
+		return xerrors.Errorf("error visiting object: %w", err)
 	}
 
 	if !visit {
-		return sz, nil
+		return nil
 	}
 
 	// occurs check -- only for DAGs
 	if c.Prefix().Codec == cid.DagCBOR {
 		has, err := s.has(c)
 		if err != nil {
-			return 0, xerrors.Errorf("error occur checking %s: %w", c, err)
+			return xerrors.Errorf("error occur checking %s: %w", c, err)
 		}
 
 		if !has {
 			err = missing(c)
 			if err == errStopWalk {
-				return sz, nil
+				return nil
 			}
 
-			return 0, err
+			return err
 		}
 	}
 
 	if err := f(c); err != nil {
 		if err == errStopWalk {
-			return sz, nil
+			return nil
 		}
 
-		return 0, err
+		return err
 	}
 
 	if c.Prefix().Codec != cid.DagCBOR {
-		return sz, nil
+		return nil
 	}
 
 	// check this before recursing
 	if err := s.checkClosing(); err != nil {
-		return sz, xerrors.Errorf("check closing: %w", err)
+		return err
 	}
 
 	var links []cid.Cid
 	err = s.view(c, func(data []byte) error {
-		sz += int64(len(data))
 		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
 			links = append(links, c)
 		})
 	})
 
 	if err != nil {
-		return 0, xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
+		return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
 	}
 
 	for _, c := range links {
-		szLink, err := s.walkObjectIncomplete(c, visitor, f, missing)
+		err := s.walkObjectIncomplete(c, visitor, f, missing)
 		if err != nil {
-			return 0, xerrors.Errorf("error walking link (cid: %s): %w", c, err)
+			return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
 		}
-		sz += szLink
 	}
 
-	return sz, nil
+	return nil
 }
 
 // internal version used during compaction and related operations
@@ -1266,10 +1065,13 @@ func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	}
 
 	err := s.hot.View(s.ctx, c, cb)
-	if ipld.IsNotFound(err) {
+	switch err {
+	case bstore.ErrNotFound:
 		return s.cold.View(s.ctx, c, cb)
+
+	default:
+		return err
 	}
-	return err
 }
 
 func (s *SplitStore) has(c cid.Cid) (bool, error) {
@@ -1288,10 +1090,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 
 func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 	blk, err := s.hot.Get(s.ctx, c)
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		return blk, nil
-	case ipld.IsNotFound(err):
+	case bstore.ErrNotFound:
 		return s.cold.Get(s.ctx, c)
 	default:
 		return nil, err
@@ -1300,10 +1102,10 @@ func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
 
 func (s *SplitStore) getSize(c cid.Cid) (int, error) {
 	sz, err := s.hot.GetSize(s.ctx, c)
-	switch {
+	switch err {
-	case err == nil:
+	case nil:
 		return sz, nil
-	case ipld.IsNotFound(err):
+	case bstore.ErrNotFound:
 		return s.cold.GetSize(s.ctx, c)
 	default:
 		return 0, err
@@ -1314,12 +1116,13 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 	batch := make([]blocks.Block, 0, batchSize)
 
 	err := coldr.ForEach(func(c cid.Cid) error {
-		if err := s.checkYield(); err != nil {
+		if err := s.checkClosing(); err != nil {
 			return err
 		}
+
 		blk, err := s.hot.Get(s.ctx, c)
 		if err != nil {
-			if ipld.IsNotFound(err) {
+			if err == bstore.ErrNotFound {
 				log.Warnf("hotstore missing block %s", c)
 				return nil
 			}
@@ -1334,7 +1137,6 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 				return xerrors.Errorf("error putting batch to coldstore: %w", err)
 			}
 			batch = batch[:0]
-
 		}
 
 		return nil
@@ -1373,21 +1175,9 @@ func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet
 		return err
 	}
 
-	now := time.Now()
-
 	err := coldr.ForEach(func(c cid.Cid) error {
 		batch = append(batch, c)
 		if len(batch) == batchSize {
-			// add some time slicing to the purge as this a very disk I/O heavy operation that
-			// requires write access to txnLk that may starve other operations that require
-			// access to the blockstore.
-			elapsed := time.Since(now)
-			if elapsed > purgeWorkSliceDuration {
-				// work 1 slice, sleep 4 slices, or 20% utilization
-				time.Sleep(4 * elapsed)
-				now = time.Now()
-			}
-
 			return deleteBatch()
 		}
 
@@ -1435,17 +1225,8 @@ func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoin
 		return 0, liveCnt, nil
 	}
 
-	switch s.compactType {
+	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
-	case hot:
+		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
-		if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
-			return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
-		}
-	case cold:
-		if err := s.cold.DeleteMany(s.ctx, deadCids); err != nil {
-			return 0, liveCnt, xerrors.Errorf("error purging dead objects: %w", err)
-		}
-	default:
-		return 0, liveCnt, xerrors.Errorf("invalid compaction type %d, only hot and cold allowed for critical section", s.compactType)
 	}
 
 	s.debug.LogDelete(deadCids)
@@ -1462,28 +1243,15 @@ func (s *SplitStore) coldSetPath() string {
 	return filepath.Join(s.path, "coldset")
 }
 
-func (s *SplitStore) discardSetPath() string {
-	return filepath.Join(s.path, "deadset")
-}
-
 func (s *SplitStore) checkpointPath() string {
 	return filepath.Join(s.path, "checkpoint")
 }
 
-func (s *SplitStore) pruneCheckpointPath() string {
-	return filepath.Join(s.path, "prune-checkpoint")
-}
-
 func (s *SplitStore) checkpointExists() bool {
 	_, err := os.Stat(s.checkpointPath())
 	return err == nil
 }
 
-func (s *SplitStore) pruneCheckpointExists() bool {
-	_, err := os.Stat(s.pruneCheckpointPath())
-	return err == nil
-}
-
 func (s *SplitStore) completeCompaction() error {
 	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
 	if err != nil {
@@ -1504,7 +1272,6 @@ func (s *SplitStore) completeCompaction() error {
 	defer markSet.Close() //nolint:errcheck
 
 	// PURGE
-	s.compactType = hot
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
 	err = s.completePurge(coldr, checkpoint, last, markSet)
@@ -1527,11 +1294,9 @@ func (s *SplitStore) completeCompaction() error {
 	if err := os.Remove(s.coldSetPath()); err != nil {
 		log.Warnf("error removing coldset: %s", err)
 	}
-	s.compactType = none
 
-	// Note: at this point we can start the splitstore; base epoch is not
+	// Note: at this point we can start the splitstore; a compaction should run on
-	//       incremented here so a compaction should run on the first head
+	//       the first head change, which will trigger gc on the hotstore.
-	//       change, which will trigger gc on the hotstore.
 	//       We don't mind the second (back-to-back) compaction as the head will
 	//       have advanced during marking and coldset accumulation.
 	return nil
@@ -1589,13 +1354,6 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint,
 	return nil
 }
 
-func (s *SplitStore) clearSizeMeasurements() {
-	s.szKeys = 0
-	s.szMarkedLiveRefs = 0
-	s.szProtectedTxns = 0
-	s.szWalk = 0
-}
-
 // I really don't like having this code, but we seem to have some occasional DAG references with
 // missing constituents. During testing in mainnet *some* of these references *sometimes* appeared
 // after a little bit.
@@ -1636,7 +1394,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
 		missing = make(map[cid.Cid]struct{})
 
 		for c := range towalk {
-			_, err := s.walkObjectIncomplete(c, visitor,
+			err := s.walkObjectIncomplete(c, visitor,
 				func(c cid.Cid) error {
 					if isUnitaryObject(c) {
 						return errStopWalk

blockstore/splitstore/splitstore_expose.go

@@ -5,8 +5,7 @@ import (
 	"errors"
 
 	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
+	cid "github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 
 	bstore "github.com/filecoin-project/lotus/blockstore"
 )
@@ -43,7 +42,6 @@ func (es *exposedSplitStore) Has(ctx context.Context, c cid.Cid) (bool, error) {
 }
 
 func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
-
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
 		if err != nil {
@@ -54,10 +52,12 @@ func (es *exposedSplitStore) Get(ctx context.Context, c cid.Cid) (blocks.Block,
 	}
 
 	blk, err := es.s.hot.Get(ctx, c)
-	if ipld.IsNotFound(err) {
+	switch err {
+	case bstore.ErrNotFound:
 		return es.s.cold.Get(ctx, c)
+	default:
+		return blk, err
 	}
-	return blk, err
 }
 
 func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
@@ -71,14 +71,12 @@ func (es *exposedSplitStore) GetSize(ctx context.Context, c cid.Cid) (int, error
 	}
 
 	size, err := es.s.hot.GetSize(ctx, c)
-	if ipld.IsNotFound(err) {
+	switch err {
+	case bstore.ErrNotFound:
 		return es.s.cold.GetSize(ctx, c)
+	default:
+		return size, err
 	}
-	return size, err
-}
-
-func (es *exposedSplitStore) Flush(ctx context.Context) error {
-	return es.s.Flush(ctx)
 }
 
 func (es *exposedSplitStore) Put(ctx context.Context, blk blocks.Block) error {
@@ -106,9 +104,11 @@ func (es *exposedSplitStore) View(ctx context.Context, c cid.Cid, f func([]byte)
 	}
 
 	err := es.s.hot.View(ctx, c, f)
-	if ipld.IsNotFound(err) {
+	switch err {
+	case bstore.ErrNotFound:
 		return es.s.cold.View(ctx, c, f)
-	}
 
-	return err
+	default:
+		return err
+	}
 }

blockstore/splitstore/splitstore_gc.go

@@ -7,96 +7,29 @@ import (
 	bstore "github.com/filecoin-project/lotus/blockstore"
 )
 
-const (
+func (s *SplitStore) gcHotstore() {
-	// Fraction of garbage in badger vlog for online GC traversal to collect garbage
-	AggressiveOnlineGCThreshold = 0.0001
-)
-
-func (s *SplitStore) gcHotAfterCompaction() {
-	// Measure hotstore size, determine if we should do full GC, determine if we can do full GC.
-	// We should do full GC if
-	//  FullGCFrequency is specified and compaction index matches frequency
-	//  OR HotstoreMaxSpaceTarget is specified and total moving space is within 150 GB of target
-	// We can do full if
-	//  HotstoreMaxSpaceTarget is not specified
-	//  OR total moving space would not exceed 50 GB below target
-	//
-	// 	a) If we should not do full GC => online GC
-	//  b) If we should do full GC and can => moving GC
-	//  c) If we should do full GC and can't => aggressive online GC
-	getSize := func() int64 {
-		sizer, ok := s.hot.(bstore.BlockstoreSize)
-		if ok {
-			size, err := sizer.Size()
-			if err != nil {
-				log.Warnf("error getting hotstore size: %s, estimating empty hot store for targeting", err)
-				return 0
-			}
-			return size
-		}
-		log.Errorf("Could not measure hotstore size, assuming it is 0 bytes, which it is not")
-		return 0
-	}
-	hotSize := getSize()
-
-	copySizeApprox := s.szKeys + s.szMarkedLiveRefs + s.szProtectedTxns + s.szWalk
-	shouldTarget := s.cfg.HotstoreMaxSpaceTarget > 0 && hotSize+copySizeApprox > int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceThreshold)
-	shouldFreq := s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0
-	shouldDoFull := shouldTarget || shouldFreq
-	canDoFull := s.cfg.HotstoreMaxSpaceTarget == 0 || hotSize+copySizeApprox < int64(s.cfg.HotstoreMaxSpaceTarget)-int64(s.cfg.HotstoreMaxSpaceSafetyBuffer)
-	log.Debugw("approximating new hot store size", "key size", s.szKeys, "marked live refs", s.szMarkedLiveRefs, "protected txns", s.szProtectedTxns, "walked DAG", s.szWalk)
-	log.Infof("measured hot store size: %d, approximate new size: %d, should do full %t, can do full %t", hotSize, copySizeApprox, shouldDoFull, canDoFull)
-
 	var opts []bstore.BlockstoreGCOption
-	if shouldDoFull && canDoFull {
+	if s.cfg.HotStoreFullGCFrequency > 0 && s.compactionIndex%int64(s.cfg.HotStoreFullGCFrequency) == 0 {
 		opts = append(opts, bstore.WithFullGC(true))
-	} else if shouldDoFull && !canDoFull {
-		log.Warnf("Attention! Estimated moving GC size %d is not within safety buffer %d of target max %d, performing aggressive online GC to attempt to bring hotstore size down safely", copySizeApprox, s.cfg.HotstoreMaxSpaceSafetyBuffer, s.cfg.HotstoreMaxSpaceTarget)
-		log.Warn("If problem continues you can 1) temporarily allocate more disk space to hotstore and 2) reflect in HotstoreMaxSpaceTarget OR trigger manual move with `lotus chain prune hot-moving`")
-		log.Warn("If problem continues and you do not have any more disk space you can run continue to manually trigger online GC at aggressive thresholds (< 0.01) with `lotus chain prune hot`")
-
-		opts = append(opts, bstore.WithThreshold(AggressiveOnlineGCThreshold))
 	}
 
 	if err := s.gcBlockstore(s.hot, opts); err != nil {
 		log.Warnf("error garbage collecting hostore: %s", err)
 	}
-	log.Infof("measured hot store size after GC: %d", getSize())
 }
 
 func (s *SplitStore) gcBlockstore(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
-	if err := s.checkYield(); err != nil {
-		return err
-	}
 	if gc, ok := b.(bstore.BlockstoreGC); ok {
 		log.Info("garbage collecting blockstore")
 		startGC := time.Now()
 
-		opts = append(opts, bstore.WithCheckFreq(90*time.Second))
+		if err := gc.CollectGarbage(opts...); err != nil {
-		opts = append(opts, bstore.WithCheck(s.checkYield))
-		if err := gc.CollectGarbage(s.ctx, opts...); err != nil {
 			return err
 		}
 
-		log.Infow("garbage collecting blockstore done", "took", time.Since(startGC))
+		log.Infow("garbage collecting hotstore done", "took", time.Since(startGC))
 		return nil
 	}
 
 	return fmt.Errorf("blockstore doesn't support garbage collection: %T", b)
 }
-
-func (s *SplitStore) gcBlockstoreOnce(b bstore.Blockstore, opts []bstore.BlockstoreGCOption) error {
-	if gc, ok := b.(bstore.BlockstoreGCOnce); ok {
-		log.Debug("gc blockstore once")
-		startGC := time.Now()
-
-		if err := gc.GCOnce(s.ctx, opts...); err != nil {
-			return err
-		}
-
-		log.Debugw("gc blockstore once done", "took", time.Since(startGC))
-		return nil
-	}
-
-	return fmt.Errorf("blockstore doesn't support gc once: %T", b)
-}

blockstore/splitstore/splitstore_prune.go

@@ -1,591 +0,0 @@
-package splitstore
-
-import (
-	"bytes"
-	"os"
-	"runtime"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	cid "github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	"go.opencensus.io/stats"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	bstore "github.com/filecoin-project/lotus/blockstore"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/metrics"
-)
-
-var (
-	// PruneOnline is a prune option that instructs PruneChain to use online gc for reclaiming space;
-	// there is no value associated with this option.
-	PruneOnlineGC = "splitstore.PruneOnlineGC"
-
-	// PruneMoving is a prune option that instructs PruneChain to use moving gc for reclaiming space;
-	// the value associated with this option is the path of the new coldstore.
-	PruneMovingGC = "splitstore.PruneMovingGC"
-
-	// PruneRetainState is a prune option that instructs PruneChain as to how many finalities worth
-	// of state to retain in the coldstore.
-	// The value is an integer:
-	// - if it is -1 then all state objects reachable from the chain will be retained in the coldstore.
-	//   this is useful for garbage collecting side-chains and other garbage in archival nodes.
-	//   This is the (safe) default.
-	// - if it is 0 then no state objects that are unreachable within the compaction boundary will
-	//   be retained in the coldstore.
-	// - if it is a positive integer, then it's the number of finalities past the compaction boundary
-	//   for which chain-reachable state objects are retained.
-	PruneRetainState = "splitstore.PruneRetainState"
-
-	// PruneThreshold is the number of epochs that need to have elapsed
-	// from the previously pruned epoch to trigger a new prune
-	PruneThreshold = 7 * build.Finality
-)
-
-// GCHotstore runs online GC on the chain state in the hotstore according the to options specified
-func (s *SplitStore) GCHotStore(opts api.HotGCOpts) error {
-	if opts.Moving {
-		gcOpts := []bstore.BlockstoreGCOption{bstore.WithFullGC(true)}
-		return s.gcBlockstore(s.hot, gcOpts)
-	}
-
-	gcOpts := []bstore.BlockstoreGCOption{bstore.WithThreshold(opts.Threshold)}
-	var err error
-	if opts.Periodic {
-		err = s.gcBlockstore(s.hot, gcOpts)
-	} else {
-		err = s.gcBlockstoreOnce(s.hot, gcOpts)
-	}
-	return err
-}
-
-// PruneChain instructs the SplitStore to prune chain state in the coldstore, according to the
-// options specified.
-func (s *SplitStore) PruneChain(opts api.PruneOpts) error {
-	retainState := opts.RetainState
-
-	var gcOpts []bstore.BlockstoreGCOption
-	if opts.MovingGC {
-		gcOpts = append(gcOpts, bstore.WithFullGC(true))
-	}
-	doGC := func() error { return s.gcBlockstore(s.cold, gcOpts) }
-
-	var retainStateP func(int64) bool
-	switch {
-	case retainState > 0:
-		retainStateP = func(depth int64) bool {
-			return depth <= int64(CompactionBoundary)+retainState*int64(build.Finality)
-		}
-	case retainState < 0:
-		retainStateP = func(_ int64) bool { return true }
-	default:
-		retainStateP = func(depth int64) bool {
-			return depth <= int64(CompactionBoundary)
-		}
-	}
-
-	if _, ok := s.cold.(bstore.BlockstoreIterator); !ok {
-		return xerrors.Errorf("coldstore does not support efficient iteration")
-	}
-
-	return s.pruneChain(retainStateP, doGC)
-}
-
-func (s *SplitStore) pruneChain(retainStateP func(int64) bool, doGC func() error) error {
-	// inhibit compaction while we are setting up
-	s.headChangeMx.Lock()
-	defer s.headChangeMx.Unlock()
-
-	// take the compaction lock; fail if there is a compaction in progress
-	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
-		return xerrors.Errorf("compaction, prune or warmup in progress")
-	}
-
-	// check if we are actually closing first
-	if atomic.LoadInt32(&s.closing) == 1 {
-		atomic.StoreInt32(&s.compacting, 0)
-		return errClosing
-	}
-
-	// ensure that we have compacted at least once
-	if s.compactionIndex == 0 {
-		atomic.StoreInt32(&s.compacting, 0)
-		return xerrors.Errorf("splitstore has not compacted yet")
-	}
-
-	// get the current tipset
-	curTs := s.chain.GetHeaviestTipSet()
-
-	// begin the transaction and go
-	s.beginTxnProtect()
-	s.compactType = cold
-	go func() {
-		defer atomic.StoreInt32(&s.compacting, 0)
-		defer s.endTxnProtect()
-
-		log.Info("pruning splitstore")
-		start := time.Now()
-
-		s.prune(curTs, retainStateP, doGC)
-
-		log.Infow("prune done", "took", time.Since(start))
-	}()
-
-	return nil
-}
-
-func (s *SplitStore) prune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) {
-	log.Debug("waiting for active views to complete")
-	start := time.Now()
-	s.viewWait()
-	log.Debugw("waiting for active views done", "took", time.Since(start))
-
-	err := s.doPrune(curTs, retainStateP, doGC)
-	if err != nil {
-		log.Errorf("PRUNE ERROR: %s", err)
-	}
-}
-
-func (s *SplitStore) doPrune(curTs *types.TipSet, retainStateP func(int64) bool, doGC func() error) error {
-	currentEpoch := curTs.Height()
-	boundaryEpoch := currentEpoch - CompactionBoundary
-
-	log.Infow("running prune", "currentEpoch", currentEpoch, "pruneEpoch", s.pruneEpoch)
-
-	markSet, err := s.markSetEnv.New("live", s.markSetSize)
-	if err != nil {
-		return xerrors.Errorf("error creating mark set: %w", err)
-	}
-	defer markSet.Close() //nolint:errcheck
-	defer s.debug.Flush()
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	// 0. track all protected references at beginning of compaction; anything added later should
-	//    be transactionally protected by the write
-	log.Info("protecting references with registered protectors")
-	err = s.applyProtectors()
-	if err != nil {
-		return err
-	}
-
-	// 1. mark reachable objects by walking the chain from the current epoch; we keep all messages
-	//    and chain headers; state and reciepts are retained only if it is within retention policy scope
-	log.Info("marking reachable objects")
-	startMark := time.Now()
-
-	count := new(int64)
-	err = s.walkChainDeep(curTs, retainStateP,
-		func(c cid.Cid) error {
-			if isUnitaryObject(c) {
-				return errStopWalk
-			}
-
-			mark, err := markSet.Has(c)
-			if err != nil {
-				return xerrors.Errorf("error checking markset: %w", err)
-			}
-
-			if mark {
-				return errStopWalk
-			}
-
-			atomic.AddInt64(count, 1)
-			return markSet.Mark(c)
-		})
-
-	if err != nil {
-		return xerrors.Errorf("error marking: %w", err)
-	}
-
-	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	// 1.1 protect transactional refs
-	err = s.protectTxnRefs(markSet)
-	if err != nil {
-		return xerrors.Errorf("error protecting transactional refs: %w", err)
-	}
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	// 2. iterate through the coldstore to collect dead objects
-	log.Info("collecting dead objects")
-	startCollect := time.Now()
-
-	deadw, err := NewColdSetWriter(s.discardSetPath())
-	if err != nil {
-		return xerrors.Errorf("error creating coldset: %w", err)
-	}
-	defer deadw.Close() //nolint:errcheck
-
-	// some stats for logging
-	var liveCnt, deadCnt int
-
-	err = s.cold.(bstore.BlockstoreIterator).ForEachKey(func(c cid.Cid) error {
-		// was it marked?
-		mark, err := markSet.Has(c)
-		if err != nil {
-			return xerrors.Errorf("error checking mark set for %s: %w", c, err)
-		}
-
-		if mark {
-			liveCnt++
-			return nil
-		}
-
-		// Note: a possible optimization here is to also purge objects that are in the hotstore
-		//       but this needs special care not to purge genesis state, so we don't bother (yet)
-
-		// it's dead in the coldstore, mark it as candidate for purge
-
-		if err := deadw.Write(c); err != nil {
-			return xerrors.Errorf("error writing cid to coldstore: %w", err)
-		}
-		deadCnt++
-
-		return nil
-	})
-
-	if err != nil {
-		return xerrors.Errorf("error dead objects: %w", err)
-	}
-
-	if err := deadw.Close(); err != nil {
-		return xerrors.Errorf("error closing deadset: %w", err)
-	}
-
-	stats.Record(s.ctx, metrics.SplitstoreCompactionDead.M(int64(deadCnt)))
-
-	log.Infow("dead collection done", "took", time.Since(startCollect))
-	log.Infow("prune stats", "live", liveCnt, "dead", deadCnt)
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	// now that we have collected dead objects, check for missing references from transactional i/o
-	// this is carried over from hot compaction for completeness
-	s.waitForMissingRefs(markSet)
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	deadr, err := NewColdSetReader(s.discardSetPath())
-	if err != nil {
-		return xerrors.Errorf("error opening deadset: %w", err)
-	}
-	defer deadr.Close() //nolint:errcheck
-
-	// 3. Purge dead objects with checkpointing for recovery.
-	// This is the critical section of prune, whereby any dead object not in the markSet is
-	// considered already deleted.
-	// We delete dead objects in batches, holding the transaction lock, where we check the markSet
-	// again for new references created by the caller.
-	// After each batch we write a checkpoint to disk; if the process is interrupted before completion
-	// the process will continue from the checkpoint in the next recovery.
-	if err := s.beginCriticalSection(markSet); err != nil {
-		return xerrors.Errorf("error beginning critical section: %w", err)
-	}
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	checkpoint, err := NewCheckpoint(s.pruneCheckpointPath())
-	if err != nil {
-		return xerrors.Errorf("error creating checkpoint: %w", err)
-	}
-	defer checkpoint.Close() //nolint:errcheck
-
-	log.Info("purging dead objects from the coldstore")
-	startPurge := time.Now()
-	err = s.purge(deadr, checkpoint, markSet)
-	if err != nil {
-		return xerrors.Errorf("error purging dead objects: %w", err)
-	}
-	log.Infow("purging dead objects from coldstore done", "took", time.Since(startPurge))
-
-	s.endCriticalSection()
-
-	if err := checkpoint.Close(); err != nil {
-		log.Warnf("error closing checkpoint: %s", err)
-	}
-	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
-		log.Warnf("error removing checkpoint: %s", err)
-	}
-	if err := deadr.Close(); err != nil {
-		log.Warnf("error closing discard set: %s", err)
-	}
-	if err := os.Remove(s.discardSetPath()); err != nil {
-		log.Warnf("error removing discard set: %s", err)
-	}
-
-	// we are done; do some housekeeping
-	s.endTxnProtect()
-	err = doGC()
-	if err != nil {
-		log.Warnf("error garbage collecting cold store: %s", err)
-	}
-
-	if err := s.setPruneEpoch(boundaryEpoch); err != nil {
-		return xerrors.Errorf("error saving prune base epoch: %w", err)
-	}
-
-	s.pruneIndex++
-	err = s.ds.Put(s.ctx, pruneIndexKey, int64ToBytes(s.pruneIndex))
-	if err != nil {
-		return xerrors.Errorf("error saving prune index: %w", err)
-	}
-
-	return nil
-}
-
-func (s *SplitStore) completePrune() error {
-	checkpoint, last, err := OpenCheckpoint(s.pruneCheckpointPath())
-	if err != nil {
-		return xerrors.Errorf("error opening checkpoint: %w", err)
-	}
-	defer checkpoint.Close() //nolint:errcheck
-
-	deadr, err := NewColdSetReader(s.discardSetPath())
-	if err != nil {
-		return xerrors.Errorf("error opening deadset: %w", err)
-	}
-	defer deadr.Close() //nolint:errcheck
-
-	markSet, err := s.markSetEnv.Recover("live")
-	if err != nil {
-		return xerrors.Errorf("error recovering markset: %w", err)
-	}
-	defer markSet.Close() //nolint:errcheck
-
-	// PURGE!
-	s.compactType = cold
-	log.Info("purging dead objects from the coldstore")
-	startPurge := time.Now()
-	err = s.completePurge(deadr, checkpoint, last, markSet)
-	if err != nil {
-		return xerrors.Errorf("error purgin dead objects: %w", err)
-	}
-	log.Infow("purging dead objects from the coldstore done", "took", time.Since(startPurge))
-
-	markSet.EndCriticalSection()
-	s.compactType = none
-
-	if err := checkpoint.Close(); err != nil {
-		log.Warnf("error closing checkpoint: %s", err)
-	}
-	if err := os.Remove(s.pruneCheckpointPath()); err != nil {
-		log.Warnf("error removing checkpoint: %s", err)
-	}
-	if err := deadr.Close(); err != nil {
-		log.Warnf("error closing deadset: %s", err)
-	}
-	if err := os.Remove(s.discardSetPath()); err != nil {
-		log.Warnf("error removing deadset: %s", err)
-	}
-
-	return nil
-}
-
-// like walkChain but peforms a deep walk, using parallel walking with walkObjectLax,
-// whereby all extant messages are retained and state roots are retained if they satisfy
-// the given predicate.
-// missing references are ignored, as we expect to have plenty for snapshot syncs.
-func (s *SplitStore) walkChainDeep(ts *types.TipSet, retainStateP func(int64) bool,
-	f func(cid.Cid) error) error {
-	visited := cid.NewSet()
-	toWalk := ts.Cids()
-	walkCnt := 0
-
-	workers := runtime.NumCPU() / 2
-	if workers < 2 {
-		workers = 2
-	}
-
-	var wg sync.WaitGroup
-	workch := make(chan cid.Cid, 16*workers)
-	errch := make(chan error, workers)
-
-	var once sync.Once
-	defer once.Do(func() { close(workch) })
-
-	push := func(c cid.Cid) error {
-		if !visited.Visit(c) {
-			return nil
-		}
-
-		select {
-		case workch <- c:
-			return nil
-		case err := <-errch:
-			return err
-		}
-	}
-
-	worker := func() {
-		defer wg.Done()
-		for c := range workch {
-			err := s.walkObjectLax(c, f)
-			if err != nil {
-				errch <- xerrors.Errorf("error walking object (cid: %s): %w", c, err)
-				return
-			}
-		}
-	}
-
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go worker()
-	}
-
-	baseEpoch := ts.Height()
-	minEpoch := baseEpoch // for progress report
-	log.Infof("walking at epoch %d", minEpoch)
-
-	walkBlock := func(c cid.Cid) error {
-		if !visited.Visit(c) {
-			return nil
-		}
-
-		walkCnt++
-
-		if err := f(c); err != nil {
-			return err
-		}
-
-		var hdr types.BlockHeader
-		err := s.view(c, func(data []byte) error {
-			return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
-		})
-
-		if err != nil {
-			return xerrors.Errorf("error unmarshaling block header (cid: %s): %w", c, err)
-		}
-
-		if hdr.Height < minEpoch {
-			minEpoch = hdr.Height
-			if minEpoch%10_000 == 0 {
-				log.Infof("walking at epoch %d (walked: %d)", minEpoch, walkCnt)
-			}
-		}
-
-		depth := int64(baseEpoch - hdr.Height)
-		retainState := retainStateP(depth)
-
-		if hdr.Height > 0 {
-			if err := push(hdr.Messages); err != nil {
-				return err
-			}
-			if retainState {
-				if err := push(hdr.ParentMessageReceipts); err != nil {
-					return err
-				}
-			}
-		}
-
-		if retainState || hdr.Height == 0 {
-			if err := push(hdr.ParentStateRoot); err != nil {
-				return err
-			}
-		}
-
-		if hdr.Height > 0 {
-			toWalk = append(toWalk, hdr.Parents...)
-		}
-
-		return nil
-	}
-
-	for len(toWalk) > 0 {
-		// walking can take a while, so check this with every opportunity
-		if err := s.checkClosing(); err != nil {
-			return err
-		}
-
-		select {
-		case err := <-errch:
-			return err
-		default:
-		}
-
-		walking := toWalk
-		toWalk = nil
-		for _, c := range walking {
-			if err := walkBlock(c); err != nil {
-				return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
-			}
-		}
-	}
-
-	once.Do(func() { close(workch) })
-	wg.Wait()
-	select {
-	case err := <-errch:
-		return err
-	default:
-	}
-
-	log.Infow("chain walk done", "walked", walkCnt)
-
-	return nil
-}
-
-// like walkObject but treats missing references laxly; faster version of walkObjectIncomplete
-// without an occurs check.
-func (s *SplitStore) walkObjectLax(c cid.Cid, f func(cid.Cid) error) error {
-	if err := f(c); err != nil {
-		if err == errStopWalk {
-			return nil
-		}
-
-		return err
-	}
-
-	if c.Prefix().Codec != cid.DagCBOR {
-		return nil
-	}
-
-	// check this before recursing
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	var links []cid.Cid
-	err := s.view(c, func(data []byte) error {
-		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
-			links = append(links, c)
-		})
-	})
-
-	if err != nil {
-		if ipld.IsNotFound(err) { // not a problem for deep walks
-			return nil
-		}
-
-		return xerrors.Errorf("error scanning linked block (cid: %s): %w", c, err)
-	}
-
-	for _, c := range links {
-		err := s.walkObjectLax(c, f)
-		if err != nil {
-			return xerrors.Errorf("error walking link (cid: %s): %w", c, err)
-		}
-	}
-
-	return nil
-}

blockstore/splitstore/splitstore_reify.go

@@ -5,9 +5,10 @@ import (
 	"runtime"
 	"sync/atomic"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
 )
 
 var (
@@ -83,7 +84,7 @@ func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
 }
 
 func (s *SplitStore) doReify(c cid.Cid) {
-	var toreify, toforget []cid.Cid
+	var toreify, totrack, toforget []cid.Cid
 
 	defer func() {
 		s.reifyMx.Lock()
@@ -92,6 +93,9 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		for _, c := range toreify {
 			delete(s.reifyInProgress, c)
 		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
 		for _, c := range toforget {
 			delete(s.reifyInProgress, c)
 		}
@@ -101,7 +105,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
 	defer s.txnLk.RUnlock()
 
 	count := 0
-	_, err := s.walkObjectIncomplete(c, newTmpVisitor(),
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
 				return errStopWalk
@@ -128,10 +132,19 @@ func (s *SplitStore) doReify(c cid.Cid) {
 				return xerrors.Errorf("error checking hotstore: %w", err)
 			}
 
-			// All reified blocks are tracked at reification start
 			if has {
-				toforget = append(toforget, c)
+				if s.txnMarkSet != nil {
-				return errStopWalk
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
 			}
 
 			toreify = append(toreify, c)
@@ -143,7 +156,7 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		})
 
 	if err != nil {
-		if errors.Is(err, errReifyLimit) {
+		if xerrors.Is(err, errReifyLimit) {
 			log.Debug("reification aborted; reify limit reached")
 			return
 		}
@@ -178,4 +191,24 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		}
 	}
 
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
 }

blockstore/splitstore/splitstore_test.go

@@ -1,30 +1,28 @@
-// stm: #unit
+//stm: #unit
 package splitstore
 
 import (
 	"context"
-	"crypto/rand"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
-	ipld "github.com/ipfs/go-ipld-format"
-	logging "github.com/ipfs/go-log/v2"
-	mh "github.com/multiformats/go-multihash"
-
 	"github.com/filecoin-project/go-state-types/abi"
-
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+	datastore "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
 )
 
 func init() {
@@ -38,7 +36,6 @@ func init() {
 func testSplitStore(t *testing.T, cfg *Config) {
 	ctx := context.Background()
 	chain := &mockChain{t: t}
-	fmt.Printf("Config: %v\n", cfg)
 
 	// the myriads of stores
 	ds := dssync.MutexWrap(datastore.NewMapDatastore())
@@ -226,7 +223,7 @@ func TestSplitStoreCompaction(t *testing.T) {
 	//stm: @SPLITSTORE_SPLITSTORE_OPEN_001, @SPLITSTORE_SPLITSTORE_CLOSE_001
 	//stm: @SPLITSTORE_SPLITSTORE_PUT_001, @SPLITSTORE_SPLITSTORE_ADD_PROTECTOR_001
 	//stm: @SPLITSTORE_SPLITSTORE_CLOSE_001
-	testSplitStore(t, &Config{MarkSetType: "map", UniversalColdBlocks: true})
+	testSplitStore(t, &Config{MarkSetType: "map"})
 }
 
 func TestSplitStoreCompactionWithBadger(t *testing.T) {
@@ -238,7 +235,7 @@ func TestSplitStoreCompactionWithBadger(t *testing.T) {
 	t.Cleanup(func() {
 		badgerMarkSetBatchSize = bs
 	})
-	testSplitStore(t, &Config{MarkSetType: "badger", UniversalColdBlocks: true})
+	testSplitStore(t, &Config{MarkSetType: "badger"})
 }
 
 func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
@@ -284,7 +281,7 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	path := t.TempDir()
 
 	// open the splitstore
-	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -423,13 +420,13 @@ func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.
 
 	path := t.TempDir()
 
-	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer ss.Close() //nolint
 
-	ss.warmupEpoch.Store(1)
+	ss.warmupEpoch = 1
 	go ss.reifyOrchestrator()
 
 	waitForReification := func() {
@@ -523,13 +520,13 @@ func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blocks
 
 	path := t.TempDir()
 
-	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map", UniversalColdBlocks: true})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer ss.Close() //nolint
 
-	ss.warmupEpoch.Store(1)
+	ss.warmupEpoch = 1
 	go ss.reifyOrchestrator()
 
 	waitForReification := func() {
@@ -699,7 +696,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 
 	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
-		return nil, ipld.ErrNotFound{Cid: cid}
+		return nil, blockstore.ErrNotFound
 	}
 	return blk, nil
 }
@@ -757,8 +754,6 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	return nil
 }
 
-func (b *mockStore) Flush(context.Context) error { return nil }
-
 func (b *mockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 	return nil, errors.New("not implemented")
 }

blockstore/splitstore/splitstore_util.go

@@ -3,10 +3,11 @@ package splitstore
 import (
 	"encoding/binary"
 
-	"github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
 
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+
 	"github.com/filecoin-project/go-state-types/abi"
 )
 

blockstore/splitstore/splitstore_warmup.go

@@ -5,13 +5,13 @@ import (
 	"sync/atomic"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-state-types/abi"
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	bstore "github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -21,14 +21,14 @@ var (
 	WarmupBoundary = build.Finality
 )
 
-// warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore;
+// warmup acuiqres the compaction lock and spawns a goroutine to warm up the hotstore;
 // this is necessary when we sync from a snapshot or when we enable the splitstore
 // on top of an existing blockstore (which becomes the coldstore).
 func (s *SplitStore) warmup(curTs *types.TipSet) error {
 	if !atomic.CompareAndSwapInt32(&s.compacting, 0, 1) {
 		return xerrors.Errorf("error locking compaction")
 	}
-	s.compactType = warmup
+
 	go func() {
 		defer atomic.StoreInt32(&s.compacting, 0)
 
@@ -88,7 +88,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 
 			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
-				if ipld.IsNotFound(err) {
+				if err == bstore.ErrNotFound {
 					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}
@@ -110,7 +110,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 			mx.Unlock()
 
 			return nil
-		}, func(cid.Cid) error { return nil })
+		})
 
 	if err != nil {
 		return err
@@ -136,8 +136,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	if err != nil {
 		return xerrors.Errorf("error saving warm up epoch: %w", err)
 	}
-
+	s.mx.Lock()
-	s.warmupEpoch.Store(int64(epoch))
+	s.warmupEpoch = epoch
+	s.mx.Unlock()
 
 	// also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes
 	err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex))

blockstore/splitstore/visitor.go

@@ -3,7 +3,7 @@ package splitstore
 import (
 	"sync"
 
-	"github.com/ipfs/go-cid"
+	cid "github.com/ipfs/go-cid"
 )
 
 // ObjectVisitor is an interface for deduplicating objects during walks

blockstore/sync.go

@@ -20,8 +20,6 @@ type SyncBlockstore struct {
 	bs MemBlockstore // specifically use a memStore to save indirection overhead.
 }
 
-func (*SyncBlockstore) Flush(context.Context) error { return nil }
-
 func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()

blockstore/timed.go

@@ -8,7 +8,6 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/raulk/clock"
 	"go.uber.org/multierr"
 )
@@ -93,16 +92,6 @@ func (t *TimedCacheBlockstore) rotate() {
 	t.mu.Unlock()
 }
 
-func (t *TimedCacheBlockstore) Flush(ctx context.Context) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-
-	if err := t.active.Flush(ctx); err != nil {
-		return err
-	}
-	return t.inactive.Flush(ctx)
-}
-
 func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error {
 	// Don't check the inactive set here. We want to keep this block for at
 	// least one interval.
@@ -123,7 +112,7 @@ func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback fun
 	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
 	block, err := t.active.Get(ctx, k)
-	if ipld.IsNotFound(err) {
+	if err == ErrNotFound {
 		block, err = t.inactive.Get(ctx, k)
 	}
 	t.mu.RUnlock()
@@ -138,7 +127,7 @@ func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	b, err := t.active.Get(ctx, k)
-	if ipld.IsNotFound(err) {
+	if err == ErrNotFound {
 		b, err = t.inactive.Get(ctx, k)
 	}
 	return b, err
@@ -148,7 +137,7 @@ func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, err
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 	size, err := t.active.GetSize(ctx, k)
-	if ipld.IsNotFound(err) {
+	if err == ErrNotFound {
 		size, err = t.inactive.GetSize(ctx, k)
 	}
 	return size, err
@@ -186,12 +175,11 @@ func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, e
 	defer t.mu.RUnlock()
 
 	ch := make(chan cid.Cid, len(t.active)+len(t.inactive))
-	for _, b := range t.active {
+	for c := range t.active {
-		ch <- b.Cid()
+		ch <- c
 	}
-	for _, b := range t.inactive {
+	for c := range t.inactive {
-		c := b.Cid()
+		if _, ok := t.active[c]; ok {
-		if _, ok := t.active[string(c.Hash())]; ok {
 			continue
 		}
 		ch <- c

blockstore/timed_test.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package blockstore
 
 import (
@@ -6,10 +6,11 @@ import (
 	"testing"
 	"time"
 
-	blocks "github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
 	"github.com/raulk/clock"
 	"github.com/stretchr/testify/require"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 )
 
 func TestTimedCacheBlockstoreSimple(t *testing.T) {

blockstore/union.go

@@ -5,16 +5,16 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	ipld "github.com/ipfs/go-ipld-format"
 )
 
 type unionBlockstore []Blockstore
 
 // Union returns an unioned blockstore.
 //
-//   - Reads return from the first blockstore that has the value, querying in the
+// * Reads return from the first blockstore that has the value, querying in the
-//     supplied order.
+//   supplied order.
-//   - Writes (puts and deletes) are broadcast to all stores.
+// * Writes (puts and deltes) are broadcast to all stores.
+//
 func Union(stores ...Blockstore) Blockstore {
 	return unionBlockstore(stores)
 }
@@ -30,7 +30,7 @@ func (m unionBlockstore) Has(ctx context.Context, cid cid.Cid) (has bool, err er
 
 func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block, err error) {
 	for _, bs := range m {
-		if blk, err = bs.Get(ctx, cid); err == nil || !ipld.IsNotFound(err) {
+		if blk, err = bs.Get(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
@@ -39,7 +39,7 @@ func (m unionBlockstore) Get(ctx context.Context, cid cid.Cid) (blk blocks.Block
 
 func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) (err error) {
 	for _, bs := range m {
-		if err = bs.View(ctx, cid, callback); err == nil || !ipld.IsNotFound(err) {
+		if err = bs.View(ctx, cid, callback); err == nil || err != ErrNotFound {
 			break
 		}
 	}
@@ -48,22 +48,13 @@ func (m unionBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]
 
 func (m unionBlockstore) GetSize(ctx context.Context, cid cid.Cid) (size int, err error) {
 	for _, bs := range m {
-		if size, err = bs.GetSize(ctx, cid); err == nil || !ipld.IsNotFound(err) {
+		if size, err = bs.GetSize(ctx, cid); err == nil || err != ErrNotFound {
 			break
 		}
 	}
 	return size, err
 }
 
-func (m unionBlockstore) Flush(ctx context.Context) (err error) {
-	for _, bs := range m {
-		if err = bs.Flush(ctx); err != nil {
-			break
-		}
-	}
-	return err
-}
-
 func (m unionBlockstore) Put(ctx context.Context, block blocks.Block) (err error) {
 	for _, bs := range m {
 		if err = bs.Put(ctx, block); err != nil {

blockstore/union_test.go

@@ -1,4 +1,4 @@
-// stm: #unit
+//stm: #unit
 package blockstore
 
 import (
@@ -13,7 +13,6 @@ var (
 	b0 = blocks.NewBlock([]byte("abc"))
 	b1 = blocks.NewBlock([]byte("foo"))
 	b2 = blocks.NewBlock([]byte("bar"))
-	b3 = blocks.NewBlock([]byte("baz"))
 )
 
 func TestUnionBlockstore_Get(t *testing.T) {

build/actors/README.md

@@ -1,26 +0,0 @@
-# Bundles
-
-This directory includes the actors bundles for each release. Each actor bundle is a zstd compressed
-tarfile containing one bundle per network type. These tarfiles are subsequently embedded in the
-lotus binary.
-
-## Updating
-
-To update, run the `./pack.sh` script. For example, the following will pack the [builtin actors release](https://github.com/filecoin-project/builtin-actors/releases) `dev/20220602` into the `v8` tarfile.
-
-```bash
-./pack.sh v8 dev/20220602
-```
-
-This will:
-
-1. Download the actors bundles and pack them into the appropriate tarfile (`$VERSION.tar.zst`).
-2. Run `make bundle-gen` in the top-level directory to regenerate the bundle metadata file for _all_ network versions (all `*.tar.zst` files in this directory).
-
-## Overriding
-
-To build a bundle, but specify a different release/tag for a specific network, append `$network=$alternative_release` on the command line. For example:
-
-```bash
-./pack.sh v8 dev/20220602 mainnet=v8.0.0 calibrationnet=v8.0.0-rc.1
-```

build/actors/pack.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-NETWORKS=(devnet mainnet caterpillarnet butterflynet testing testing-fake-proofs calibrationnet)
-
-set -e
-
-if [[ $# -lt 2 ]]; then
-    echo "Usage: $0 VERSION RELEASE [NETWORK=RELEASE_OVERRIDE]..." >&2
-    echo "expected at least two arguments, an actors version (e.g., v8), an actors release, and any number of release overrides." >&2
-    exit 1
-fi
-
-VERSION="$1" # actors version
-RELEASE="$2" # actors release name
-RELEASE_OVERRIDES=("${@:3}")
-
-echo "Downloading bundles for actors version ${VERSION} release ${RELEASE}"
-echo "With release overrides ${RELEASE_OVERRIDES[*]}"
-
-TARGET_FILE="$(pwd)/${VERSION}.tar.zst"
-WORKDIR=$(mktemp -d -t "actor-bundles-${VERSION}.XXXXXXXXXX")
-trap 'rm -rf -- "$WORKDIR"' EXIT
-
-encode_release() {
-    jq -rn --arg release "$1" '$release | @uri'
-}
-
-pushd "${WORKDIR}"
-for network in "${NETWORKS[@]}"; do
-    release="$RELEASE"
-    # Ideally, we'd use an associative array (map). But that's not supported on macos.
-    for override in "${RELEASE_OVERRIDES[@]}"; do
-        if [[ "${network}" = "${override%%=*}" ]]; then
-            release="${override#*=}"
-            break
-        fi
-    done
-    encoded_release="$(encode_release "$release")"
-    echo "Downloading $release for network $network."
-    wget "https://github.com/filecoin-project/builtin-actors/releases/download/${encoded_release}/builtin-actors-${network}"{.car,.sha256}
-done
-
-echo "Checking the checksums..."
-
-sha256sum -c -- *.sha256
-
-echo "Packing..."
-
-rm -f -- "$TARGET_FILE"
-tar -cf "$TARGET_FILE" --use-compress-program "zstd -19" -- *.car
-popd
-
-echo "Generating metadata..."
-
-make -C ../../ VERSION="$VERSION" RELEASE="$RELEASE" RELEASE_OVERRIDES="${RELEASE_OVERRIDES[*]}" bundle-gen