remove debug

chore: releases: merge releases branch to master

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 orbs:
-  aws-cli: circleci/aws-cli@1.3.2
-  docker: circleci/docker@2.1.4
+  aws-cli: circleci/aws-cli@4.1.1
+  docker: circleci/docker@2.3.0
 
 executors:
   golang:
@@ -70,8 +70,6 @@ commands:
           name: Restore parameters cache
           keys:
             - 'v26-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
       - run: ./lotus fetch-params 2048
       - save_cache:
           name: Save parameters cache
@@ -96,6 +94,7 @@ commands:
             git fetch --all
   install-ubuntu-deps:
     steps:
+      - run: sudo apt install curl ca-certificates gnupg
       - run: sudo apt-get update
       - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
   check-go-version:
@@ -143,9 +142,9 @@ jobs:
       Run tests with gotestsum.
     working_directory: ~/lotus
     parameters: &test-params
-      executor:
-        type: executor
-        default: golang
+      resource_class:
+        type: string
+        default: medium+
       go-test-flags:
         type: string
         default: "-timeout 20m"
@@ -164,7 +163,14 @@ jobs:
         type: string
         default: unit
         description: Test suite name to report to CircleCI.
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:1.20
+        environment:
+          LOTUS_HARMONYDB_HOSTS: yugabyte
+      - image: yugabytedb/yugabyte:2.18.0.0-b65
+        command: bin/yugabyted start --daemon=false
+        name: yugabyte
+    resource_class: << parameters.resource_class >>
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -182,6 +188,8 @@ jobs:
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
+            dockerize -wait tcp://yugabyte:5433 -timeout 3m
+            env
             gotestsum \
               --format standard-verbose \
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
@@ -209,7 +217,9 @@ jobs:
           Branch on github.com/filecoin-project/test-vectors to checkout and
           test with. If empty (the default) the commit defined by the git
           submodule is used.
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:1.20
+    resource_class: << parameters.resource_class >>
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -396,15 +406,14 @@ jobs:
       Run golangci-lint.
     working_directory: ~/lotus
     parameters:
-      executor:
-        type: executor
-        default: golang
       args:
         type: string
         default: ''
         description: |
           Arguments to pass to golangci-lint
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:1.20
+    resource_class: medium+
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -575,7 +584,7 @@ workflows:
             - build
           suite: itest-deals_concurrent
           target: "./itests/deals_concurrent_test.go"
-          executor: golang-2xl
+          resource_class: 2xlarge
       - test:
           name: test-itest-deals_invalid_utf8_label
           requires:
@@ -768,6 +777,18 @@ workflows:
             - build
           suite: itest-get_messages_in_ts
           target: "./itests/get_messages_in_ts_test.go"
+      - test:
+          name: test-itest-harmonydb
+          requires:
+            - build
+          suite: itest-harmonydb
+          target: "./itests/harmonydb_test.go"
+      - test:
+          name: test-itest-harmonytask
+          requires:
+            - build
+          suite: itest-harmonytask
+          target: "./itests/harmonytask_test.go"
       - test:
           name: test-itest-lite_migration
           requires:
@@ -858,12 +879,6 @@ workflows:
             - build
           suite: itest-pending_deal_allocation
           target: "./itests/pending_deal_allocation_test.go"
-      - test:
-          name: test-itest-raft_messagesigner
-          requires:
-            - build
-          suite: itest-raft_messagesigner
-          target: "./itests/raft_messagesigner_test.go"
       - test:
           name: test-itest-remove_verifreg_datacap
           requires:
@@ -976,14 +991,14 @@ workflows:
             - build
           suite: itest-wdpost_worker_config
           target: "./itests/wdpost_worker_config_test.go"
-          executor: golang-2xl
+          resource_class: 2xlarge
       - test:
           name: test-itest-worker
           requires:
             - build
           suite: itest-worker
           target: "./itests/worker_test.go"
-          executor: golang-2xl
+          resource_class: 2xlarge
       - test:
           name: test-itest-worker_upgrade
           requires:
@@ -996,7 +1011,7 @@ workflows:
             - build
           suite: utest-unit-cli
           target: "./cli/... ./cmd/... ./api/..."
-          executor: golang-2xl
+          resource_class: 2xlarge
           get-params: true
       - test:
           name: test-unit-node
@@ -1010,7 +1025,7 @@ workflows:
             - build
           suite: utest-unit-rest
           target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
-          executor: golang-2xl
+          resource_class: 2xlarge
       - test:
           name: test-unit-storage
           requires:

.circleci/gen.go

@@ -10,11 +10,25 @@ import (
 	"text/template"
 )
 
+var GoVersion = "" // from init below. Ex: 1.19.7
+
 //go:generate go run ./gen.go ..
 
 //go:embed template.yml
 var templateFile embed.FS
 
+func init() {
+	b, err := os.ReadFile("../go.mod")
+	if err != nil {
+		panic("cannot find go.mod in parent folder")
+	}
+	for _, line := range strings.Split(string(b), "\n") {
+		if strings.HasPrefix(line, "go ") {
+			GoVersion = line[3:]
+		}
+	}
+}
+
 type (
 	dirs  = []string
 	suite = string
@@ -111,6 +125,7 @@ func main() {
 		Networks   []string
 		ItestFiles []string
 		UnitSuites map[string]string
+		GoVersion  string
 	}
 	in := data{
 		Networks:   []string{"mainnet", "butterflynet", "calibnet", "debug"},
@@ -125,6 +140,7 @@ func main() {
 			}
 			return ret
 		}(),
+		GoVersion: GoVersion,
 	}
 
 	out, err := os.Create("./config.yml")

.circleci/template.yml

@@ -1,7 +1,7 @@
 version: 2.1
 orbs:
-  aws-cli: circleci/aws-cli@1.3.2
-  docker: circleci/docker@2.1.4
+  aws-cli: circleci/aws-cli@4.1.1
+  docker: circleci/docker@2.3.0
 
 executors:
   golang:
@@ -70,8 +70,6 @@ commands:
           name: Restore parameters cache
           keys:
             - 'v26-2k-lotus-params'
-          paths:
-            - /var/tmp/filecoin-proof-parameters/
       - run: ./lotus fetch-params 2048
       - save_cache:
           name: Save parameters cache
@@ -96,6 +94,7 @@ commands:
             git fetch --all
   install-ubuntu-deps:
     steps:
+      - run: sudo apt install curl ca-certificates gnupg
       - run: sudo apt-get update
       - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
   check-go-version:
@@ -143,9 +142,9 @@ jobs:
       Run tests with gotestsum.
     working_directory: ~/lotus
     parameters: &test-params
-      executor:
-        type: executor
-        default: golang
+      resource_class:
+        type: string
+        default: medium+
       go-test-flags:
         type: string
         default: "-timeout 20m"
@@ -164,7 +163,14 @@ jobs:
         type: string
         default: unit
         description: Test suite name to report to CircleCI.
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:[[ .GoVersion]]
+        environment:
+          LOTUS_HARMONYDB_HOSTS: yugabyte
+      - image: yugabytedb/yugabyte:2.18.0.0-b65
+        command: bin/yugabyted start --daemon=false
+        name: yugabyte
+    resource_class: << parameters.resource_class >>
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -182,6 +188,8 @@ jobs:
           command: |
             mkdir -p /tmp/test-reports/<< parameters.suite >>
             mkdir -p /tmp/test-artifacts
+            dockerize -wait tcp://yugabyte:5433 -timeout 3m
+            env
             gotestsum \
               --format standard-verbose \
               --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
@@ -209,7 +217,9 @@ jobs:
           Branch on github.com/filecoin-project/test-vectors to checkout and
           test with. If empty (the default) the commit defined by the git
           submodule is used.
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:[[ .GoVersion]]
+    resource_class: << parameters.resource_class >>
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -396,15 +406,14 @@ jobs:
       Run golangci-lint.
     working_directory: ~/lotus
     parameters:
-      executor:
-        type: executor
-        default: golang
       args:
         type: string
         default: ''
         description: |
           Arguments to pass to golangci-lint
-    executor: << parameters.executor >>
+    docker:
+      - image: cimg/go:[[ .GoVersion]]
+    resource_class: medium+
     steps:
       - install-ubuntu-deps
       - attach_workspace:
@@ -543,7 +552,7 @@ workflows:
           suite: itest-[[ $name ]]
           target: "./itests/[[ $file ]]"
           [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]]
-          executor: golang-2xl
+          resource_class: 2xlarge
           [[- end]]
           [[- if or (eq $name "wdpost") (eq $name "sector_pledge")]]
           get-params: true
@@ -561,11 +570,11 @@ workflows:
           get-params: true
           [[- end -]]
           [[- if eq $suite "unit-cli"]]
-          executor: golang-2xl
+          resource_class: 2xlarge
           get-params: true
           [[- end -]]
           [[- if eq $suite "unit-rest"]]
-          executor: golang-2xl
+          resource_class: 2xlarge
           [[- end -]]
       [[- end]]
       - test:

.gitignore

@@ -6,6 +6,7 @@
 /lotus-chainwatch
 /lotus-shed
 /lotus-sim
+/lotus-provider
 /lotus-townhall
 /lotus-fountain
 /lotus-stats

CHANGELOG.md

@@ -1,21 +1,200 @@
 # Lotus changelog
 
 # UNRELEASED
-- chore: Auto remove local chain data when importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
-- feat: metric: export Mpool message count ([filecoin-project/lotus#11361](https://github.com/filecoin-project/lotus/pull/11361))
-- feat: sealing: load SectorsSummary from sealing SectorStats instead of calling API each time ([filecoin-project/lotus#11353](https://github.com/filecoin-project/lotus/pull/11353))
-- fix: miner info: Show correct sector state counts ([filecoin-project/lotus#11456](https://github.com/filecoin-project/lotus/pull/11456))
-- feat: add support for specifying block number when calling `eth_estimateGas` ([filecoin-project/lotus#11462](https://github.com/filecoin-project/lotus/pull/11462)).
 
 ## Improvements
-- fix: Add time slicing to splitstore purging step during compaction to reduce lock congestion [filecoin-project/lotus#11269](https://github.com/filecoin-project/lotus/pull/11269)
-- feat: Added instructions on how to setup Prometheus/Grafana for monitoring a local Lotus node [filecoin-project/lotus#11276](https://github.com/filecoin-project/lotus/pull/11276)
-- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
-- fix: The Ethereum API will now use the correct state-tree when resolving "native" addresses into masked ID addresses. Additionally, pending messages from native account types won't be visible in the Ethereum API because there is no "correct" state-tree to pick in this case. However, pending _Ethereum_ transactions and native messages that have landed on-chain will still be visible through the Ethereum API.
-- feat: Unambiguously translate native messages to Ethereum transactions by:
-  - Detecting native messages that "look" like Ethereum transactions (creating smart contracts, invoking a smart contract, etc.), and decoding them as such.
-  - Otherwise, ABI-encoding the inputs as if they were calls to a `handle_filecoin_method` Solidity method.
-- fix: ensure that the Ethereum API never returns "empty" addresses for native messages. When a "to" address cannot be resolved to a 0x-style address, it will be re-written to `0xff0000000000000000000000ffffffffffffffff`. This can only happen when the native transaction _reverted_ (failing to create an account at the specified "to" address).
+
+# v1.25.2 / 2024-01-11 
+
+This is an optional but **highly recommended feature release** of Lotus, as it includes fixes for synchronizations issues that users have experienced. The feature release also introduces `Lotus-Provider` in its alpha testing phase, as well as the ability to call external PC2-binaries during the sealing process.
+
+## ☢️ Upgrade Warnings ☢️
+
+There are no upgrade warnings for this feature release.
+
+## ⭐️ Highlights ⭐️
+
+### Lotus-Provider
+The feature release ships the alpha release of the new Lotus-Provider binary, together with its initial features - High Availability of WindowPoSt and WinningPoSt.
+
+So what is so exciting about Lotus-Provider:
+
+**High Availability**
+- You can run as many `Lotus-Provider` instances as you want for both WindowPoSt and WinningPOSt.
+- You can connect them to as many clustered Yugabyte instances as you want to. This allows for an NxN configuration where all instances can communicate with all others.
+- You have the option to connect different instances to different chain daemons.
+
+**Simplicity**
+- Once the configuration is in the database, setting up a new machine with Lotus-Provider is straightforward. Simply start the binary with the correct flags to find YugabyteDB and specify which configuration layers it should use.
+
+**Durability**
+- `Lotus-Provider` is designed with robustness in mind. Updates to the system are handled seamlessly, ensuring that performance and stability are maintained when taking down machines in your cluster for updates.
+
+Read more about [`Lotus-Provider` in the documentation here](https://lotus.filecoin.io/storage-providers/lotus-provider/overview/). And check out the how you can migrate from [Lotus-Miner to Lotus-Provider here](https://lotus.filecoin.io/storage-providers/lotus-provider/setup/). **(Only recommended in testnets while its in Alpha)**
+
+### External PC2-binaries 
+
+In this feature release, storage providers can call external PC2-binaries during the sealing process. This allows storage providers to leverage the SupraSeal PC2 binary, which has been shown to improve sealing speed in the PC2-phase. For instance, our current benchmarks show that an NVIDIA RTX A5000 card was able to complete PC2 in approximately 2.5 minutes.
+
+We have verified that SupraSeal PC2 functions properly with Committed Capacity (CC) sectors, both SyntheticPoReps and non-Synthetic PoReps. However calling SupraSeal PC2 with deal sectors is not supported in this feature release.
+
+For more information on how to use SupraSeal PC2 with your `lotus-worker`, as well as how to use feature, please [refer to the documentation](https://lotus.filecoin.io/tutorials/lotus-miner/supra-seal-pc2/).
+
+## New features
+- feat: sturdypost work branch ([filecoin-project/lotus#11405](https://github.com/filecoin-project/lotus/pull/11405))
+   - Adds the `Lotus-Provider` binary, and the HarmonyDB framework.
+- feat: worker: Support delegating precommit2 to external binary ([filecoin-project/lotus#11185](https://github.com/filecoin-project/lotus/pull/11185))
+   - Allows for delegating PreCommit2 to an exteranl binary.
+- feat: build: Add SupraSeal-PC2 binary script ([filecoin-project/lotus#11430](https://github.com/filecoin-project/lotus/pull/11430))
+   - Adds a script for building the SupraSeal-PC2 binary easily.
+- Feat: daemon: Auto remove existing chain if importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
+   - Auto removes the existing chain when importing a snapshot. 
+- feat: Add ETA to lotus sync wait (#11211) ([filecoin-project/lotus#11211](https://github.com/filecoin-project/lotus/pull/11211))
+   - Adds a ETA indicator to `lotus sync wait`, so you can get an estimate for how long until sync is completed.
+- feat: mpool/wdpost: Maximize feecap config ([filecoin-project/lotus#9746](https://github.com/filecoin-project/lotus/pull/9746))
+   - Adds a Maximixe FeeCap Config
+- feat: Add lotus-bench cli option to stress test any binary ([filecoin-project/lotus#11270](https://github.com/filecoin-project/lotus/pull/11270))
+   - Enables the `Lotus-Bench` to run any binary and analyze their latency and histogram distribution, track most common errors, perform stress testing under different concurrency levels and see how it works under different QPS.
+- feat: chain import: don't walk to genesis - 2-3x faster snapshot import (#11446) ([filecoin-project/lotus#11446](https://github.com/filecoin-project/lotus/pull/11446))
+   - Improves Snapshot import speed, by not walking back to genesis on import.
+- feat: metric: export Mpool message count ([filecoin-project/lotus#11361](https://github.com/filecoin-project/lotus/pull/11361))
+   - Adds the mpool count as a prometheus metric.
+- feat: bench: flag to output GenerateWinningPoStWithVanilla params ([filecoin-project/lotus#11460](https://github.com/filecoin-project/lotus/pull/11460))
+
+## Improvements
+- feat: bootstrap: add glif bootstrap node on calibration ([filecoin-project/lotus#11175](https://github.com/filecoin-project/lotus/pull/11175))
+- fix: bench: Set ticket and seed to a non-all zero value ([filecoin-project/lotus#11429](https://github.com/filecoin-project/lotus/pull/11429))
+- fix: alert: Check UDPbuffer-size ([filecoin-project/lotus#11360](https://github.com/filecoin-project/lotus/pull/11360))
+- feat: cli: sort actor CIDs alphabetically before printing (#11345) ([filecoin-project/lotus#11345](https://github.com/filecoin-project/lotus/pull/11345))
+- fix: worker: Connect when --listen is not set ([filecoin-project/lotus#11294](https://github.com/filecoin-project/lotus/pull/11294))
+- fix: miner info: Show correct sector state counts ([filecoin-project/lotus#11456](https://github.com/filecoin-project/lotus/pull/11456))
+- feat: miner: defensive check for equivocation ([filecoin-project/lotus#11321](https://github.com/filecoin-project/lotus/pull/11321))
+- feat: Instructions for setting up Grafana/Prometheus for monitoring local lotus node ([filecoin-project/lotus#11276](https://github.com/filecoin-project/lotus/pull/11276))
+- fix: cli: Wrap error in wallet sign ([filecoin-project/lotus#11273](https://github.com/filecoin-project/lotus/pull/11273))
+- fix: Add time slicing to splitstore purging to reduce lock congestion ([filecoin-project/lotus#11269](https://github.com/filecoin-project/lotus/pull/11269))
+- feat: sealing: load SectorsSummary from sealing SectorStats instead of calling API each time ([filecoin-project/lotus#11353](https://github.com/filecoin-project/lotus/pull/11353))
+- fix: shed: additional metrics in `mpool miner-select-messages` ([filecoin-project/lotus#11253](https://github.com/filecoin-project/lotus/pull/11253))
+- storage: Return soft err when sector alloc fails in acquire ([filecoin-project/lotus#11338](https://github.com/filecoin-project/lotus/pull/11338))
+- feat: miner: log detailed timing breakdown when mining takes longer than the block's timestamp ([filecoin-project/lotus#11228](https://github.com/filecoin-project/lotus/pull/11228))
+- fix: shed: make invariants checker work with splitstore ([filecoin-project/lotus#11391](https://github.com/filecoin-project/lotus/pull/11391))
+- feat: eth: encode eth tx input as solidity ABI (#11402) ([filecoin-project/lotus#11402](https://github.com/filecoin-project/lotus/pull/11402))
+- fix: eth: use the correct state-tree when resolving addresses (#11387) ([filecoin-project/lotus#11387](https://github.com/filecoin-project/lotus/pull/11387))
+- fix: eth: remove trace sanity check (#11385) ([filecoin-project/lotus#11385](https://github.com/filecoin-project/lotus/pull/11385))
+- fix: chain: make failure to load the chain state fatal (#11426) ([filecoin-project/lotus#11426](https://github.com/filecoin-project/lotus/pull/11426))
+- fix: build: an epoch is near an upgrade iff the upgrade is enabled (#11401) ([filecoin-project/lotus#11401](https://github.com/filecoin-project/lotus/pull/11401))
+- fix: eth: handle unresolvable addresses (#11433) ([filecoin-project/lotus#11433](https://github.com/filecoin-project/lotus/pull/11433))
+- fix: eth: correctly encode and simplify native input/output encoding (#11382) ([filecoin-project/lotus#11382](https://github.com/filecoin-project/lotus/pull/11382))
+- fix: worker: listen for interrupt signals in GetStorageMinerAPI loop (#11309) ([filecoin-project/lotus#11309](https://github.com/filecoin-project/lotus/pull/11309))
+- fix: sync: iterate over returned messages directly (#11373) ([filecoin-project/lotus#11373](https://github.com/filecoin-project/lotus/pull/11373))
+- fix: miner: correct duration logs in mineOne ([filecoin-project/lotus#11241](https://github.com/filecoin-project/lotus/pull/11241))
+- fix: cli: Add print to unseal cmd ([filecoin-project/lotus#11271](https://github.com/filecoin-project/lotus/pull/11271))
+- fix: networking: avoid dialing when trying to handshake peers ([filecoin-project/lotus#11262](https://github.com/filecoin-project/lotus/pull/11262))
+- metric milliseconds computation with golang original method (#11403) ([filecoin-project/lotus#11403](https://github.com/filecoin-project/lotus/pull/11403))
+- feat: shed: fix blockstore prune (#11197) ([filecoin-project/lotus#11197](https://github.com/filecoin-project/lotus/pull/11197))
+- refactor:ffi: replace ClearLayerData with ClearCache (#11352) ([filecoin-project/lotus#11352](https://github.com/filecoin-project/lotus/pull/11352))
+- fix: api: compute gasUsedRatio based on max gas in the tipset (#11354) ([filecoin-project/lotus#11354](https://github.com/filecoin-project/lotus/pull/11354))
+- fix: api: compute the effective gas cost with the correct base-fee (#11357) ([filecoin-project/lotus#11357](https://github.com/filecoin-project/lotus/pull/11357))
+- fix: api: return errors on failure to lookup an eth txn receipt (#11329) ([filecoin-project/lotus#11329](https://github.com/filecoin-project/lotus/pull/11329))
+- fix: api: exclude reverted events in `eth_getLogs` results (#11318) ([filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318))
+- api: Add block param to eth_estimateGas ([filecoin-project/lotus#11462](https://github.com/filecoin-project/lotus/pull/11462))
+- opt: fix duplicate check exitcode ([filecoin-project/lotus#11171](https://github.com/filecoin-project/lotus/pull/11171))
+- fix: lotus-provider: show addresses in log ([filecoin-project/lotus#11490](https://github.com/filecoin-project/lotus/pull/11490))
+- fix: lotus-provider: Wait for the correct taskID ([filecoin-project/lotus#11493](https://github.com/filecoin-project/lotus/pull/11493))
+- harmony: Fix task reclaim on restart ([filecoin-project/lotus#11498](https://github.com/filecoin-project/lotus/pull/11498))
+- fix: lotus-provider: Fix log output format in wdPostTaskCmd ([filecoin-project/lotus#11504](https://github.com/filecoin-project/lotus/pull/11504))
+- fix: lp docsgen ([filecoin-project/lotus#11488](https://github.com/filecoin-project/lotus/pull/11488))
+- fix: lotus-provider do not suggest default layer ([filecoin-project/lotus#11486](https://github.com/filecoin-project/lotus/pull/11486))
+- feat: syncer: optimize syncFork for one-epoch forks ([filecoin-project/lotus#11533](https://github.com/filecoin-project/lotus/pull/11533))
+- fix: sync: do not include incoming in return of syncFork ([filecoin-project/lotus#11541](https://github.com/filecoin-project/lotus/pull/11541))
+- fix: wdpost: fix vanilla proof indexes ([filecoin-project/lotus#11550](https://github.com/filecoin-project/lotus/pull/11550))
+- feat: exchange: change GetBlocks to always fetch the requested number of tipsets ([filecoin-project/lotus#11565](https://github.com/filecoin-project/lotus/pull/11565))
+
+## Dependencies
+- update go-libp2p to v0.31.0 ([filecoin-project/lotus#11225](https://github.com/filecoin-project/lotus/pull/11225))
+- deps: gostatetype (#11437) ([filecoin-project/lotus#11437](https://github.com/filecoin-project/lotus/pull/11437))
+- fix: deps: stop using go-libp2p deprecated peer.ID.Pretty ([filecoin-project/lotus#11263](https://github.com/filecoin-project/lotus/pull/11263))
+- chore:libp2p:update libp2p deps in release-v1.25.2 to v0.31.1 ([filecoin-project/lotus#11524](https://github.com/filecoin-project/lotus/pull/11524))
+- deps: update go-multiaddr to v0.12.0 ([filecoin-project/lotus#11524](https://github.com/filecoin-project/lotus/pull/11558))
+- dep: go-multi-address to v0.12.1 ([filecoin-project/lotus#11564](https://github.com/filecoin-project/lotus/pull/11564))
+
+## Others
+- chore: update FFI (#11431) ([filecoin-project/lotus#11431](https://github.com/filecoin-project/lotus/pull/11431))
+- chore: build: bump master to v1.25.1-dev ([filecoin-project/lotus#11450](https://github.com/filecoin-project/lotus/pull/11450))
+- chore: releases :merge releases into master  ([filecoin-project/lotus#11448](https://github.com/filecoin-project/lotus/pull/11448))
+- chore: actors: update v12 to the final release ([filecoin-project/lotus#11440](https://github.com/filecoin-project/lotus/pull/11440))
+- chore: Remove ipfs main bootstrap nodes (#11200) ([filecoin-project/lotus#11200](https://github.com/filecoin-project/lotus/pull/11200))
+- Remove PL's european bootstrap nodes from mainnet.pi ([filecoin-project/lotus#11315](https://github.com/filecoin-project/lotus/pull/11315))
+- chore: deps: update to go-state-types v0.12.7 ([filecoin-project/lotus#11428](https://github.com/filecoin-project/lotus/pull/11428))
+- fix: Add .vscode to gitignore ([filecoin-project/lotus#11275](https://github.com/filecoin-project/lotus/pull/11275))
+- fix: test: temporarily exempt SynthPorep constants from test ([filecoin-project/lotus#11259](https://github.com/filecoin-project/lotus/pull/11259))
+- feat: skip TestSealAndVerify3 until it's fixed ([filecoin-project/lotus#11230](https://github.com/filecoin-project/lotus/pull/11230))
+- Update RELEASE_ISSUE_TEMPLATE.md ([filecoin-project/lotus#11250](https://github.com/filecoin-project/lotus/pull/11250))
+- fix: config: Update ColdStoreType comments ([filecoin-project/lotus#11274](https://github.com/filecoin-project/lotus/pull/11274))
+- readme: bump up golang version (#11347) ([filecoin-project/lotus#11347](https://github.com/filecoin-project/lotus/pull/11347))
+- chore: watermelon: upgrade epoch ([filecoin-project/lotus#11374](https://github.com/filecoin-project/lotus/pull/11374))
+- add support for v12 check invariants and also a default case to reduce future confusion (#11371) ([filecoin-project/lotus#11371](https://github.com/filecoin-project/lotus/pull/11371))
+- test: drand: switch tests to drand testnet (from devnet) (#11359) ([filecoin-project/lotus#11359](https://github.com/filecoin-project/lotus/pull/11359))
+- feat: chain: light-weight patch to fix calibrationnet again by removing move_partitions from built-in actors (#11409) ([filecoin-project/lotus#11409](https://github.com/filecoin-project/lotus/pull/11409))
+- chore: cli: Revert move-partitions cmd ([filecoin-project/lotus#11408](https://github.com/filecoin-project/lotus/pull/11408))
+- chore: forward-port calibnet hotfix to master ([filecoin-project/lotus#11407](https://github.com/filecoin-project/lotus/pull/11407))
+- fix: migration: set premigration to 90 minutes ([filecoin-project/lotus#11395](https://github.com/filecoin-project/lotus/pull/11395))
+- feat: chain: light-weight patch to fix calibrationnet (#11363) ([filecoin-project/lotus#11363](https://github.com/filecoin-project/lotus/pull/11363))
+- chore: merge feat/nv21 into master ([filecoin-project/lotus#11336](https://github.com/filecoin-project/lotus/pull/11336))
+- docs: Link the release section in the release flow doc ([filecoin-project/lotus#11299](https://github.com/filecoin-project/lotus/pull/11299))
+- fix: ci: fetch params for the storage unit tests ([filecoin-project/lotus#11441](https://github.com/filecoin-project/lotus/pull/11441))
+- Update mainnet.pi ([filecoin-project/lotus#11288](https://github.com/filecoin-project/lotus/pull/11288))
+- add go linter - "unused" (#11235) ([filecoin-project/lotus#11235](https://github.com/filecoin-project/lotus/pull/11235))
+- Fix/texts (#11298) ([filecoin-project/lotus#11298](https://github.com/filecoin-project/lotus/pull/11298))
+- fix typo in rate-limit flag description (#11316) ([filecoin-project/lotus#11316](https://github.com/filecoin-project/lotus/pull/11316))
+- eth_filter flake debug ([filecoin-project/lotus#11261](https://github.com/filecoin-project/lotus/pull/11261))
+- fix: sealing: typo in FinalizeReplicaUpdate ([filecoin-project/lotus#11255](https://github.com/filecoin-project/lotus/pull/11255))
+- chore: slice loop replace (#11349) ([filecoin-project/lotus#11349](https://github.com/filecoin-project/lotus/pull/11349))
+- backport: docker build fix for v1.25.2 ([filecoin-project/lotus#11560](https://github.com/filecoin-project/lotus/pull/11560))
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Andrew Jackson (Ajax) | 161 | +24328/-12464 | 4148 |
+| Łukasz Magiera | 99 | +5238/-2690 | 260 |
+| Shrenuj Bansal | 27 | +3402/-1265 | 76 |
+| Fridrik Asmundsson | 15 | +1148/-307 | 58 |
+| Steven Allen | 15 | +674/-337 | 35 |
+| Ian Norden | 1 | +625/-3 | 4 |
+| Aarsh Shah | 4 | +227/-167 | 14 |
+| Phi | 19 | +190/-183 | 32 |
+| Aayush Rajasekaran | 3 | +291/-56 | 16 |
+| Mikers | 2 | +76/-262 | 19 |
+| Aayush | 14 | +111/-59 | 21 |
+| Friðrik Ásmundsson | 1 | +101/-1 | 2 |
+| Alejandro Criado-Pérez | 1 | +36/-36 | 27 |
+| Jie Hou | 5 | +36/-10 | 5 |
+| Florian RUEN | 2 | +24/-19 | 5 |
+| Phi-rjan | 3 | +20/-8 | 3 |
+| Icarus9913 | 1 | +11/-11 | 6 |
+| Jiaying Wang | 3 | +8/-7 | 5 |
+| guangwu | 1 | +3/-10 | 2 |
+| Marten Seemann | 1 | +6/-6 | 2 |
+| simlecode | 1 | +0/-6 | 2 |
+| GlacierWalrus | 2 | +0/-5 | 2 |
+| Anton Evangelatov | 1 | +2/-2 | 1 |
+| Ales Dumikau | 3 | +2/-2 | 3 |
+| renran | 1 | +2/-1 | 1 |
+| Volker Mische | 1 | +1/-1 | 1 |
+| Icarus Wu | 1 | +1/-1 | 1 |
+| Hubert | 1 | +1/-1 | 1 |
+| Aloxaf | 1 | +1/-1 | 1 |
+| Alejandro | 1 | +1/-1 | 1 |
+| lazavikmaria | 1 | +1/-0 | 1 |
+
+# v1.25.1 / 2023-12-09
+
+This is a **highly recommended PATCH RELEASE.**  The patch release fixes the issue were node operators trying to catch up sync were unable to sync large message blocks/epochs due to an increased number of messages on the network.
+
+This patch release allows for up to 10k messages per block. Additionally, it introduces a limit on the amount of data that can be read at once, ensuring the system can handle worst-case scenarios.
+
+## Improvements
+- fix: exchange: allow up to 10k messages per block ([filecoin-project/lotus#11506](https://github.com/filecoin-project/lotus/pull/11506))
 
 # v 1.25.0 / 2023-11-22
 
@@ -28,7 +207,7 @@ The full list of [protocol improvements delivered in the network upgrade can be
 ## ☢️ Upgrade Warnings ☢️
 
 - Read through the [changelog of the mandatory v1.24.0 release](https://github.com/filecoin-project/lotus/releases/tag/v1.24.0). Especially the `Migration` and `v12 Builtin Actor Bundle` sections.
-- Please remove and clone a new Lotus repo (`git clone https://github.com/filecoin-project/lotus.git`) when upgrading to this release. 
+- Please remove and clone a new Lotus repo (`git clone https://github.com/filecoin-project/lotus.git`) when upgrading to this release.
 - This feature release requires a minimum Go version of v1.20.7 or higher to successfully build Lotus. Go version 1.21.x is not supported yet.
 - EthRPC providers, please check out the [new tracing API to Lotus RPC](https://github.com/filecoin-project/lotus/pull/11100)
 
@@ -101,6 +280,9 @@ Lotus-workers can now be built to leverage the SupraSeal C2 sealing optimization
 - fix(client): single-root error message ([filecoin-project/lotus#11214](https://github.com/filecoin-project/lotus/pull/11214))
 - fix: worker: Convert `DC_[SectorSize]_[ResourceRestriction]` if set ([filecoin-project/lotus#11224](https://github.com/filecoin-project/lotus/pull/11224))
 - chore: backport #11338 onto release/v1.25.0 ([filecoin-project/lotus#11350](https://github.com/filecoin-project/lotus/pull/11350))
+- fix: lotus-provider: lotus-provider msg sending ([filecoin-project/lotus#11480](https://github.com/filecoin-project/lotus/pull/11480))
+- fix: lotus-provider: Fix winning PoSt ([filecoin-project/lotus#11483](https://github.com/filecoin-project/lotus/pull/11483))
+- chore: fix: sql Scan cannot write to an object ([filecoin-project/lotus#11487](https://github.com/filecoin-project/lotus/pull/11487))
 
 ## Dependencies
 - deps: update go-libp2p to v0.28.1 ([filecoin-project/lotus#10998](https://github.com/filecoin-project/lotus/pull/10998))
@@ -190,7 +372,7 @@ account           bafk2bzaceboftg75mdiba7xbo2i3uvgtca4brhnr3u5ptihonixgpnrvhpxoa
 init              bafk2bzacebllyegx5r6lggf6ymyetbp7amacwpuxakhtjvjtvoy2bfkzk3vms
 ```
 
-## Migration  
+## Migration
 
 We are expecting a heavier than normal state migration for this upgrade due to the amount of state changes introduced for miner sector info. (This is a similar migration as the Shark upgrade, however, we have introduced a couple of migration performance optimizations since then for a smoother upgrade experience.)
 
@@ -209,7 +391,7 @@ You can check out the [tutorial for benchmarking the network migration here.](ht
 
 ## BREAKING CHANGE
 
-There is a new protocol limit on how many partition could be submited in one PoSt - if you have any customized tooling for batching PoSts, please update accordingly. 
+There is a new protocol limit on how many partition could be submited in one PoSt - if you have any customized tooling for batching PoSts, please update accordingly.
 - feat: limit PoSted partitions to 3 ([filecoin-project/lotus#11327](https://github.com/filecoin-project/lotus/pull/11327))
 
 ## New features
@@ -221,7 +403,7 @@ There is a new protocol limit on how many partition could be submited in one PoS
    
 ## Improvements
 - Backport: feat: sealing: Switch to calling PreCommitSectorBatch2 ([filecoin-project/lotus#11215](https://github.com/filecoin-project/lotus/pull/11215))
-- updated the boostrap nodes 
+- updated the boostrap nodes
 
 ## Dependencies
 - github.com/filecoin-project/go-amt-ipld/v4 (v4.0.0 -> v4.2.0)
@@ -231,9 +413,9 @@ There is a new protocol limit on how many partition could be submited in one PoS
 - chore: deps: update libp2p to v0.30.0 #11434
 
 
-## Snapshots 
+## Snapshots
 
-The [Forest team](https://filecoinproject.slack.com/archives/C029LPZ5N73) at Chainsafe has launched a brand new lightweight snapshot service that is backed up by forest nodes! This is a great alternative service along with the fil-infra one, and it is compatible with lotus! We recommend lotus users to check it out [here](https://docs.filecoin.io/networks/mainnet#resources)! 
+The [Forest team](https://filecoinproject.slack.com/archives/C029LPZ5N73) at Chainsafe has launched a brand new lightweight snapshot service that is backed up by forest nodes! This is a great alternative service along with the fil-infra one, and it is compatible with lotus! We recommend lotus users to check it out [here](https://docs.filecoin.io/networks/mainnet#resources)!
 
 
 

Dockerfile

@@ -109,6 +109,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-wallet   /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-gateway  /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-miner    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-worker   /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-provider    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
 COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
 
@@ -117,11 +118,13 @@ RUN mkdir /var/lib/lotus
 RUN mkdir /var/lib/lotus-miner
 RUN mkdir /var/lib/lotus-worker
 RUN mkdir /var/lib/lotus-wallet
+RUN mkdir /var/lib/lotus-provider
 RUN chown fc: /var/tmp/filecoin-proof-parameters
 RUN chown fc: /var/lib/lotus
 RUN chown fc: /var/lib/lotus-miner
 RUN chown fc: /var/lib/lotus-worker
 RUN chown fc: /var/lib/lotus-wallet
+RUN chown fc: /var/lib/lotus-provider
 
 
 VOLUME /var/tmp/filecoin-proof-parameters
@@ -129,6 +132,7 @@ VOLUME /var/lib/lotus
 VOLUME /var/lib/lotus-miner
 VOLUME /var/lib/lotus-worker
 VOLUME /var/lib/lotus-wallet
+VOLUME /var/lib/lotus-provider
 
 EXPOSE 1234
 EXPOSE 2345

Makefile

@@ -66,7 +66,7 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
-build-devnets: build lotus-seed lotus-shed
+build-devnets: build lotus-seed lotus-shed lotus-provider
 .PHONY: build-devnets
 
 debug: GOFLAGS+=-tags=debug
@@ -97,6 +97,15 @@ lotus-miner: $(BUILD_DEPS)
 .PHONY: lotus-miner
 BINS+=lotus-miner
 
+lotus-provider: $(BUILD_DEPS)
+	rm -f lotus-provider
+	$(GOCC) build $(GOFLAGS) -o lotus-provider ./cmd/lotus-provider
+.PHONY: lotus-provider
+BINS+=lotus-provider
+
+lp2k: GOFLAGS+=-tags=2k
+lp2k: lotus-provider
+
 lotus-worker: $(BUILD_DEPS)
 	rm -f lotus-worker
 	$(GOCC) build $(GOFLAGS) -o lotus-worker ./cmd/lotus-worker
@@ -115,13 +124,13 @@ lotus-gateway: $(BUILD_DEPS)
 .PHONY: lotus-gateway
 BINS+=lotus-gateway
 
-build: lotus lotus-miner lotus-worker
+build: lotus lotus-miner lotus-worker 
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
 an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
+install: install-daemon install-miner install-worker install-provider
 
 install-daemon:
 	install -C ./lotus /usr/local/bin/lotus
@@ -129,6 +138,9 @@ install-daemon:
 install-miner:
 	install -C ./lotus-miner /usr/local/bin/lotus-miner
 
+install-provider:
+	install -C ./lotus-provider /usr/local/bin/lotus-provider
+
 install-worker:
 	install -C ./lotus-worker /usr/local/bin/lotus-worker
 
@@ -144,6 +156,9 @@ uninstall-daemon:
 uninstall-miner:
 	rm -f /usr/local/bin/lotus-miner
 
+uninstall-provider:
+	rm -f /usr/local/bin/lotus-provider
+
 uninstall-worker:
 	rm -f /usr/local/bin/lotus-worker
 
@@ -241,6 +256,14 @@ install-miner-service: install-miner install-daemon-service
 	@echo
 	@echo "lotus-miner service installed. Don't forget to run 'sudo systemctl start lotus-miner' to start it and 'sudo systemctl enable lotus-miner' for it to be enabled on startup."
 
+install-provider-service: install-provider install-daemon-service
+	mkdir -p /etc/systemd/system
+	mkdir -p /var/log/lotus
+	install -C -m 0644 ./scripts/lotus-provider.service /etc/systemd/system/lotus-provider.service
+	systemctl daemon-reload
+	@echo
+	@echo "lotus-provider service installed. Don't forget to run 'sudo systemctl start lotus-provider' to start it and 'sudo systemctl enable lotus-provider' for it to be enabled on startup."
+
 install-main-services: install-miner-service
 
 install-all-services: install-main-services
@@ -259,6 +282,12 @@ clean-miner-service:
 	rm -f /etc/systemd/system/lotus-miner.service
 	systemctl daemon-reload
 
+clean-provider-service:
+	-systemctl stop lotus-provider
+	-systemctl disable lotus-provider
+	rm -f /etc/systemd/system/lotus-provider.service
+	systemctl daemon-reload
+
 clean-main-services: clean-daemon-service
 
 clean-all-services: clean-main-services
@@ -294,7 +323,8 @@ actors-code-gen:
 	$(GOCC) run ./chain/actors/agen
 	$(GOCC) fmt ./...
 
-actors-gen: actors-code-gen fiximports
+actors-gen: actors-code-gen 
+	./scripts/fiximports
 .PHONY: actors-gen
 
 bundle-gen:
@@ -328,7 +358,7 @@ docsgen-md-bin: api-gen actors-gen
 docsgen-openrpc-bin: api-gen actors-gen
 	$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
 
-docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
+docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker docsgen-md-provider
 
 docsgen-md-full: docsgen-md-bin
 	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
@@ -337,6 +367,8 @@ docsgen-md-storage: docsgen-md-bin
 	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
 docsgen-md-worker: docsgen-md-bin
 	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
+docsgen-md-provider: docsgen-md-bin
+	./docgen-md "api/api_lp.go" "Provider" "api" "./api" > documentation/en/api-v0-methods-provider.md
 
 docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
 
@@ -354,21 +386,23 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin
 fiximports:
 	./scripts/fiximports
 
-gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci fiximports
+gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci
+	./scripts/fiximports
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'"
 .PHONY: gen
 
 jen: gen
 
-snap: lotus lotus-miner lotus-worker
+snap: lotus lotus-miner lotus-worker lotus-provider
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
 
 # separate from gen because it needs binaries
-docsgen-cli: lotus lotus-miner lotus-worker
+docsgen-cli: lotus lotus-miner lotus-worker lotus-provider
 	python3 ./scripts/generate-lotus-cli.py
 	./lotus config default > documentation/en/default-lotus-config.toml
 	./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
+	./lotus-provider config default > documentation/en/default-lotus-provider-config.toml
 .PHONY: docsgen-cli
 
 print-%:

api/api_full.go

@@ -879,9 +879,6 @@ type FullNode interface {
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
-
-	RaftState(ctx context.Context) (*RaftStateData, error) //perm:read
-	RaftLeader(ctx context.Context) (peer.ID, error)       //perm:read
 }
 
 // reverse interface to the client, called after EthSubscribe

api/api_lp.go

@@ -0,0 +1,10 @@
+package api
+
+import "context"
+
+type LotusProvider interface {
+	Version(context.Context) (Version, error) //perm:admin
+
+	// Trigger shutdown
+	Shutdown(context.Context) error //perm:admin
+}

api/client/client.go

@@ -15,6 +15,16 @@ import (
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 )
 
+// NewProviderRpc creates a new http jsonrpc client.
+func NewProviderRpc(ctx context.Context, addr string, requestHeader http.Header) (api.LotusProvider, jsonrpc.ClientCloser, error) {
+	var res v1api.LotusProviderStruct
+
+	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
+		api.GetInternalStructs(&res), requestHeader, jsonrpc.WithErrors(api.RPCErrors))
+
+	return &res, closer, err
+}
+
 // NewCommonRPCV0 creates a new http jsonrpc client.
 func NewCommonRPCV0(ctx context.Context, addr string, requestHeader http.Header) (api.CommonNet, jsonrpc.ClientCloser, error) {
 	var res v0api.CommonNetStruct

api/docgen/docgen.go

@@ -355,10 +355,6 @@ func init() {
 	addExample(map[string]bitfield.BitField{
 		"": bitfield.NewFromSet([]uint64{5, 6, 7, 10}),
 	})
-	addExample(&api.RaftStateData{
-		NonceMap: make(map[address.Address]uint64),
-		MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
-	})
 
 	addExample(http.Header{
 		"Authorization": []string{"Bearer ey.."},
@@ -432,6 +428,10 @@ func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []r
 			i = &api.GatewayStruct{}
 			t = reflect.TypeOf(new(struct{ api.Gateway })).Elem()
 			permStruct = append(permStruct, reflect.TypeOf(api.GatewayStruct{}.Internal))
+		case "Provider":
+			i = &api.LotusProviderStruct{}
+			t = reflect.TypeOf(new(struct{ api.LotusProvider })).Elem()
+			permStruct = append(permStruct, reflect.TypeOf(api.LotusProviderStruct{}.Internal))
 		default:
 			panic("unknown type")
 		}

api/mocks/mock_full.go

@@ -2919,36 +2919,6 @@ func (mr *MockFullNodeMockRecorder) PaychVoucherSubmit(arg0, arg1, arg2, arg3, a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PaychVoucherSubmit", reflect.TypeOf((*MockFullNode)(nil).PaychVoucherSubmit), arg0, arg1, arg2, arg3, arg4)
 }
 
-// RaftLeader mocks base method.
-func (m *MockFullNode) RaftLeader(arg0 context.Context) (peer.ID, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RaftLeader", arg0)
-	ret0, _ := ret[0].(peer.ID)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// RaftLeader indicates an expected call of RaftLeader.
-func (mr *MockFullNodeMockRecorder) RaftLeader(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RaftLeader", reflect.TypeOf((*MockFullNode)(nil).RaftLeader), arg0)
-}
-
-// RaftState mocks base method.
-func (m *MockFullNode) RaftState(arg0 context.Context) (*api.RaftStateData, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "RaftState", arg0)
-	ret0, _ := ret[0].(*api.RaftStateData)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// RaftState indicates an expected call of RaftState.
-func (mr *MockFullNodeMockRecorder) RaftState(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RaftState", reflect.TypeOf((*MockFullNode)(nil).RaftState), arg0)
-}
-
 // Session mocks base method.
 func (m *MockFullNode) Session(arg0 context.Context) (uuid.UUID, error) {
 	m.ctrl.T.Helper()

api/permissioned.go

@@ -41,6 +41,12 @@ func PermissionedWorkerAPI(a Worker) Worker {
 	return &out
 }
 
+func PermissionedAPI[T, P any](a T) *P {
+	var out P
+	permissionedProxies(a, &out)
+	return &out
+}
+
 func PermissionedWalletAPI(a Wallet) Wallet {
 	var out WalletStruct
 	permissionedProxies(a, &out)

api/proxy_gen.go

@@ -453,10 +453,6 @@ type FullNodeMethods struct {
 
 	PaychVoucherSubmit func(p0 context.Context, p1 address.Address, p2 *paych.SignedVoucher, p3 []byte, p4 []byte) (cid.Cid, error) `perm:"sign"`
 
-	RaftLeader func(p0 context.Context) (peer.ID, error) `perm:"read"`
-
-	RaftState func(p0 context.Context) (*RaftStateData, error) `perm:"read"`
-
 	StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) `perm:"read"`
 
 	StateActorCodeCIDs func(p0 context.Context, p1 abinetwork.Version) (map[string]cid.Cid, error) `perm:"read"`
@@ -831,6 +827,19 @@ type GatewayMethods struct {
 type GatewayStub struct {
 }
 
+type LotusProviderStruct struct {
+	Internal LotusProviderMethods
+}
+
+type LotusProviderMethods struct {
+	Shutdown func(p0 context.Context) error `perm:"admin"`
+
+	Version func(p0 context.Context) (Version, error) `perm:"admin"`
+}
+
+type LotusProviderStub struct {
+}
+
 type NetStruct struct {
 	Internal NetMethods
 }
@@ -3223,28 +3232,6 @@ func (s *FullNodeStub) PaychVoucherSubmit(p0 context.Context, p1 address.Address
 	return *new(cid.Cid), ErrNotSupported
 }
 
-func (s *FullNodeStruct) RaftLeader(p0 context.Context) (peer.ID, error) {
-	if s.Internal.RaftLeader == nil {
-		return *new(peer.ID), ErrNotSupported
-	}
-	return s.Internal.RaftLeader(p0)
-}
-
-func (s *FullNodeStub) RaftLeader(p0 context.Context) (peer.ID, error) {
-	return *new(peer.ID), ErrNotSupported
-}
-
-func (s *FullNodeStruct) RaftState(p0 context.Context) (*RaftStateData, error) {
-	if s.Internal.RaftState == nil {
-		return nil, ErrNotSupported
-	}
-	return s.Internal.RaftState(p0)
-}
-
-func (s *FullNodeStub) RaftState(p0 context.Context) (*RaftStateData, error) {
-	return nil, ErrNotSupported
-}
-
 func (s *FullNodeStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported
@@ -5214,6 +5201,28 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) {
 	return "", ErrNotSupported
 }
 
+func (s *LotusProviderStruct) Shutdown(p0 context.Context) error {
+	if s.Internal.Shutdown == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.Shutdown(p0)
+}
+
+func (s *LotusProviderStub) Shutdown(p0 context.Context) error {
+	return ErrNotSupported
+}
+
+func (s *LotusProviderStruct) Version(p0 context.Context) (Version, error) {
+	if s.Internal.Version == nil {
+		return *new(Version), ErrNotSupported
+	}
+	return s.Internal.Version(p0)
+}
+
+func (s *LotusProviderStub) Version(p0 context.Context) (Version, error) {
+	return *new(Version), ErrNotSupported
+}
+
 func (s *NetStruct) ID(p0 context.Context) (peer.ID, error) {
 	if s.Internal.ID == nil {
 		return *new(peer.ID), ErrNotSupported
@@ -7442,6 +7451,7 @@ var _ CommonNet = new(CommonNetStruct)
 var _ EthSubscriber = new(EthSubscriberStruct)
 var _ FullNode = new(FullNodeStruct)
 var _ Gateway = new(GatewayStruct)
+var _ LotusProvider = new(LotusProviderStruct)
 var _ Net = new(NetStruct)
 var _ Signable = new(SignableStruct)
 var _ StorageMiner = new(StorageMinerStruct)

api/types.go

@@ -69,11 +69,6 @@ type MessageSendSpec struct {
 	MaximizeFeeCap bool
 }
 
-type MpoolMessageWhole struct {
-	Msg  *types.Message
-	Spec *MessageSendSpec
-}
-
 // GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
 type GraphSyncDataTransfer struct {
 	// GraphSync request id for this transfer
@@ -351,64 +346,6 @@ type ForkUpgradeParams struct {
 	UpgradeWatermelonHeight  abi.ChainEpoch
 }
 
-type NonceMapType map[address.Address]uint64
-type MsgUuidMapType map[uuid.UUID]*types.SignedMessage
-
-type RaftStateData struct {
-	NonceMap NonceMapType
-	MsgUuids MsgUuidMapType
-}
-
-func (n *NonceMapType) MarshalJSON() ([]byte, error) {
-	marshalled := make(map[string]uint64)
-	for a, n := range *n {
-		marshalled[a.String()] = n
-	}
-	return json.Marshal(marshalled)
-}
-
-func (n *NonceMapType) UnmarshalJSON(b []byte) error {
-	unmarshalled := make(map[string]uint64)
-	err := json.Unmarshal(b, &unmarshalled)
-	if err != nil {
-		return err
-	}
-	*n = make(map[address.Address]uint64)
-	for saddr, nonce := range unmarshalled {
-		a, err := address.NewFromString(saddr)
-		if err != nil {
-			return err
-		}
-		(*n)[a] = nonce
-	}
-	return nil
-}
-
-func (m *MsgUuidMapType) MarshalJSON() ([]byte, error) {
-	marshalled := make(map[string]*types.SignedMessage)
-	for u, msg := range *m {
-		marshalled[u.String()] = msg
-	}
-	return json.Marshal(marshalled)
-}
-
-func (m *MsgUuidMapType) UnmarshalJSON(b []byte) error {
-	unmarshalled := make(map[string]*types.SignedMessage)
-	err := json.Unmarshal(b, &unmarshalled)
-	if err != nil {
-		return err
-	}
-	*m = make(map[uuid.UUID]*types.SignedMessage)
-	for suid, msg := range unmarshalled {
-		u, err := uuid.Parse(suid)
-		if err != nil {
-			return err
-		}
-		(*m)[u] = msg
-	}
-	return nil
-}
-
 // ChainExportConfig holds configuration for chain ranged exports.
 type ChainExportConfig struct {
 	WriteBufferSize   int

api/v1api/latest.go

@@ -12,3 +12,5 @@ type RawFullNodeAPI FullNode
 func PermissionedFullAPI(a FullNode) FullNode {
 	return api.PermissionedFullAPI(a)
 }
+
+type LotusProviderStruct = api.LotusProviderStruct

api/version.go

@@ -59,6 +59,8 @@ var (
 
 	MinerAPIVersion0  = newVer(1, 5, 0)
 	WorkerAPIVersion0 = newVer(1, 7, 0)
+
+	ProviderAPIVersion0 = newVer(1, 0, 0)
 )
 
 //nolint:varcheck,deadcode

blockstore/splitstore/splitstore_test.go

@@ -3,9 +3,9 @@ package splitstore
 
 import (
 	"context"
+	"crypto/rand"
 	"errors"
 	"fmt"
-	"math/rand"
 	"sync"
 	"sync/atomic"
 	"testing"

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.25.1-dev"
+const BuildVersion = "1.25.3-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/policy/policy.go

@@ -867,6 +867,24 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base
 	}
 }
 
+var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof
+
+func init() {
+	PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof)
+	for sealProof, info := range abi.SealProofInfos {
+		PoStToSealMap[info.WinningPoStProof] = sealProof
+		PoStToSealMap[info.WindowPoStProof] = sealProof
+	}
+}
+
+func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
+	sealProof, exists := PoStToSealMap[postProof]
+	if !exists {
+		return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof")
+	}
+	return sealProof, nil
+}
+
 func min(a, b int) int {
 	if a < b {
 		return a

chain/actors/policy/policy.go.template

@@ -343,9 +343,26 @@ func AggregatePreCommitNetworkFee(nwVer network.Version, aggregateSize int, base
 	}
 }
 
+var PoStToSealMap map[abi.RegisteredPoStProof]abi.RegisteredSealProof
+func init() {
+	PoStToSealMap = make(map[abi.RegisteredPoStProof]abi.RegisteredSealProof)
+	for sealProof, info := range abi.SealProofInfos {
+		PoStToSealMap[info.WinningPoStProof] = sealProof
+		PoStToSealMap[info.WindowPoStProof] = sealProof
+	}
+}
+
+func GetSealProofFromPoStProof(postProof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) {
+	sealProof, exists := PoStToSealMap[postProof]
+	if !exists {
+		return 0, xerrors.New("no corresponding RegisteredSealProof for the given RegisteredPoStProof")
+	}
+	return sealProof, nil
+}
+
 func min(a, b int) int {
     if a < b {
         return a
     }
     return b
-}
+}

chain/consensus/common.go

@@ -362,7 +362,8 @@ func CreateBlockHeader(ctx context.Context, sm *stmgr.StateManager, pts *types.T
 	var blsMsgCids, secpkMsgCids []cid.Cid
 	var blsSigs []crypto.Signature
 	nv := sm.GetNetworkVersion(ctx, bt.Epoch)
-	for _, msg := range bt.Messages {
+	for _, msgTmp := range bt.Messages {
+		msg := msgTmp
 		if msg.Signature.Type == crypto.SigTypeBLS {
 			blsSigs = append(blsSigs, msg.Signature)
 			blsMessages = append(blsMessages, &msg.Message)

chain/exchange/cbor_gen.go

@@ -306,9 +306,9 @@ func (t *Response) UnmarshalCBOR(r io.Reader) (err error) {
 	return nil
 }
 
-var lengthBufCompactedMessages = []byte{132}
+var lengthBufCompactedMessagesCBOR = []byte{132}
 
-func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
@@ -316,12 +316,12 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write(lengthBufCompactedMessages); err != nil {
+	if _, err := cw.Write(lengthBufCompactedMessagesCBOR); err != nil {
 		return err
 	}
 
 	// t.Bls ([]*types.Message) (slice)
-	if len(t.Bls) > cbg.MaxLength {
+	if len(t.Bls) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Bls was too long")
 	}
 
@@ -334,7 +334,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 	if len(t.BlsIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
 	}
@@ -343,24 +343,13 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.BlsIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 
 	// t.Secpk ([]*types.SignedMessage) (slice)
-	if len(t.Secpk) > cbg.MaxLength {
+	if len(t.Secpk) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Secpk was too long")
 	}
 
@@ -373,7 +362,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 	if len(t.SecpkIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
 	}
@@ -382,26 +371,15 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.SecpkIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 	return nil
 }
 
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = CompactedMessages{}
+func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = CompactedMessagesCBOR{}
 
 	cr := cbg.NewCborReader(r)
 
@@ -430,7 +408,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Bls: array too large (%d)", extra)
 	}
 
@@ -471,7 +449,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -487,7 +465,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.BlsIncludes = make([][]uint64, extra)
+		t.BlsIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -499,47 +477,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.BlsIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.BlsIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.BlsIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.BlsIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 
@@ -550,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Secpk: array too large (%d)", extra)
 	}
 
@@ -591,7 +535,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -607,7 +551,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.SecpkIncludes = make([][]uint64, extra)
+		t.SecpkIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -619,47 +563,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.SecpkIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.SecpkIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.SecpkIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.SecpkIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 

chain/exchange/client.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"time"
 
@@ -23,6 +24,10 @@ import (
 	"github.com/filecoin-project/lotus/lib/peermgr"
 )
 
+// Set the max exchange message size to 120MiB. Purely based on gas numbers, we can include ~8MiB of
+// messages per block, so I've set this to 120MiB to be _very_ safe.
+const maxExchangeMessageSize = (15 * 8) << 20
+
 // client implements exchange.Client, using the libp2p ChainExchange protocol
 // as the fetching mechanism.
 type client struct {
@@ -279,16 +284,18 @@ func (c *client) validateCompressedIndices(chain []*BSTipSet) error {
 				len(msgs.SecpkIncludes), blocksNum)
 		}
 
+		blsLen := uint64(len(msgs.Bls))
+		secpLen := uint64(len(msgs.Secpk))
 		for blockIdx := 0; blockIdx < blocksNum; blockIdx++ {
 			for _, mi := range msgs.BlsIncludes[blockIdx] {
-				if int(mi) >= len(msgs.Bls) {
+				if mi >= blsLen {
 					return xerrors.Errorf("index in BlsIncludes (%d) exceeds number of messages (%d)",
 						mi, len(msgs.Bls))
 				}
 			}
 
 			for _, mi := range msgs.SecpkIncludes[blockIdx] {
-				if int(mi) >= len(msgs.Secpk) {
+				if mi >= secpLen {
 					return xerrors.Errorf("index in SecpkIncludes (%d) exceeds number of messages (%d)",
 						mi, len(msgs.Secpk))
 				}
@@ -310,18 +317,36 @@ func (c *client) GetBlocks(ctx context.Context, tsk types.TipSetKey, count int)
 		)
 	}
 
-	req := &Request{
-		Head:    tsk.Cids(),
-		Length:  uint64(count),
-		Options: Headers,
+	var ret []*types.TipSet
+	start := tsk.Cids()
+	for len(ret) < count {
+		req := &Request{
+			Head:    start,
+			Length:  uint64(count - len(ret)),
+			Options: Headers,
+		}
+
+		validRes, err := c.doRequest(ctx, req, nil, nil)
+		if err != nil {
+			return nil, xerrors.Errorf("failed to doRequest: %w", err)
+		}
+
+		if len(validRes.tipsets) == 0 {
+			return nil, xerrors.Errorf("doRequest fetched zero tipsets: %w", err)
+		}
+
+		ret = append(ret, validRes.tipsets...)
+
+		last := validRes.tipsets[len(validRes.tipsets)-1]
+		if last.Height() <= 1 {
+			// we've walked all the way up to genesis, return
+			break
+		}
+
+		start = last.Parents().Cids()
 	}
 
-	validRes, err := c.doRequest(ctx, req, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return validRes.tipsets, nil
+	return ret, nil
 }
 
 // GetFullTipSet implements Client.GetFullTipSet(). Refer to the godocs there.
@@ -336,12 +361,16 @@ func (c *client) GetFullTipSet(ctx context.Context, peer peer.ID, tsk types.TipS
 
 	validRes, err := c.doRequest(ctx, req, &peer, nil)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("failed to doRequest: %w", err)
 	}
 
-	return validRes.toFullTipSets()[0], nil
-	// If `doRequest` didn't fail we are guaranteed to have at least
-	//  *one* tipset here, so it's safe to index directly.
+	fullTipsets := validRes.toFullTipSets()
+
+	if len(fullTipsets) == 0 {
+		return nil, xerrors.New("unexpectedly got no tipsets in exchange")
+	}
+
+	return fullTipsets[0], nil
 }
 
 // GetChainMessages implements Client.GetChainMessages(). Refer to the godocs there.
@@ -434,10 +463,11 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 		log.Warnw("CloseWrite err", "error", err)
 	}
 
-	// Read response.
+	// Read response, limiting the size of the response to maxExchangeMessageSize as we allow a
+	// lot of messages (10k+) but they'll mostly be quite small.
 	var res Response
 	err = cborutil.ReadCborRPC(
-		bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)),
+		bufio.NewReader(io.LimitReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline), maxExchangeMessageSize)),
 		&res)
 	if err != nil {
 		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)

chain/exchange/interfaces.go

@@ -28,8 +28,8 @@ type Server interface {
 // used by the Syncer.
 type Client interface {
 	// GetBlocks fetches block headers from the network, from the provided
-	// tipset *backwards*, returning as many tipsets as the count parameter,
-	// or less.
+	// tipset *backwards*, returning as many tipsets as the count parameter.
+	// The ONLY case in which we return fewer than `count` tipsets is if we hit genesis.
 	GetBlocks(ctx context.Context, tsk types.TipSetKey, count int) ([]*types.TipSet, error)
 
 	// GetChainMessages fetches messages from the network, starting from the first provided tipset

chain/exchange/protocol.go

@@ -154,6 +154,8 @@ type BSTipSet struct {
 // FIXME: The logic to decompress this structure should belong
 //
 //	to itself, not to the consumer.
+//
+// NOTE: Max messages is: BlockMessageLimit (10k) * MaxTipsetSize (15) = 150k
 type CompactedMessages struct {
 	Bls         []*types.Message
 	BlsIncludes [][]uint64

chain/exchange/protocol_encoding.go

@@ -0,0 +1,125 @@
+package exchange
+
+import (
+	"fmt"
+	"io"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/build"
+	types "github.com/filecoin-project/lotus/chain/types"
+)
+
+// Type used for encoding/decoding compacted messages. This is a ustom type as we need custom limits.
+// - Max messages is 150,000 as that's 15 times the max block size (in messages). It needs to be
+// large enough to cover a full tipset full of full blocks.
+type CompactedMessagesCBOR struct {
+	Bls         []*types.Message `cborgen:"maxlen=150000"`
+	BlsIncludes []messageIndices
+
+	Secpk         []*types.SignedMessage `cborgen:"maxlen=150000"`
+	SecpkIncludes []messageIndices
+}
+
+// Unmarshal into the "decoding" struct, then copy into the actual struct.
+func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
+	var c CompactedMessagesCBOR
+	if err := c.UnmarshalCBOR(r); err != nil {
+		return err
+	}
+	t.Bls = c.Bls
+	t.BlsIncludes = make([][]uint64, len(c.BlsIncludes))
+	for i, v := range c.BlsIncludes {
+		t.BlsIncludes[i] = v.v
+	}
+	t.Secpk = c.Secpk
+	t.SecpkIncludes = make([][]uint64, len(c.SecpkIncludes))
+	for i, v := range c.SecpkIncludes {
+		t.SecpkIncludes[i] = v.v
+	}
+	return nil
+}
+
+// Copy into the encoding struct, then marshal.
+func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	var c CompactedMessagesCBOR
+	c.Bls = t.Bls
+	c.BlsIncludes = make([]messageIndices, len(t.BlsIncludes))
+	for i, v := range t.BlsIncludes {
+		c.BlsIncludes[i].v = v
+	}
+	c.Secpk = t.Secpk
+	c.SecpkIncludes = make([]messageIndices, len(t.SecpkIncludes))
+	for i, v := range t.SecpkIncludes {
+		c.SecpkIncludes[i].v = v
+	}
+	return c.MarshalCBOR(w)
+}
+
+// this needs to be a struct or cborgen will peak into it and ignore the Unmarshal/Marshal functions
+type messageIndices struct {
+	v []uint64
+}
+
+func (t *messageIndices) UnmarshalCBOR(r io.Reader) (err error) {
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra > uint64(build.BlockMessageLimit) {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	if extra > 0 {
+		t.v = make([]uint64, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+		maj, extra, err := cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.v[i] = extra
+
+	}
+	return nil
+}
+
+func (t *messageIndices) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if len(t.v) > build.BlockMessageLimit {
+		return xerrors.Errorf("Slice value in field v was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.v))); err != nil {
+		return err
+	}
+	for _, v := range t.v {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, v); err != nil {
+			return err
+		}
+	}
+	return nil
+}

chain/gen/genesis/miners.go

@@ -251,7 +251,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 			}
 
 			params := &markettypes.PublishStorageDealsParams{}
-			for _, preseal := range m.Sectors {
+			for _, presealTmp := range m.Sectors {
+				preseal := presealTmp
 				preseal.Deal.VerifiedDeal = true
 				preseal.Deal.EndEpoch = minerInfos[i].presealExp
 				p := markettypes.ClientDealProposal{

chain/messagepool/block_proba_test.go

@@ -5,7 +5,6 @@ import (
 	"math"
 	"math/rand"
 	"testing"
-	"time"
 )
 
 func TestBlockProbability(t *testing.T) {
@@ -23,7 +22,6 @@ func TestBlockProbability(t *testing.T) {
 
 func TestWinnerProba(t *testing.T) {
 	//stm: @OTHER_IMPLEMENTATION_BLOCK_PROB_002
-	rand.Seed(time.Now().UnixNano())
 	const N = 1000000
 	winnerProba := noWinnersProb()
 	sum := 0

chain/messagesigner/messagesigner_consensus.go

@@ -1,98 +0,0 @@
-package messagesigner
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagepool"
-	"github.com/filecoin-project/lotus/chain/types"
-	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-type MessageSignerConsensus struct {
-	MsgSigner
-	Consensus *consensus.Consensus
-}
-
-func NewMessageSignerConsensus(
-	wallet api.Wallet,
-	mpool messagepool.MpoolNonceAPI,
-	ds dtypes.MetadataDS,
-	consensus *consensus.Consensus) *MessageSignerConsensus {
-
-	ds = namespace.Wrap(ds, datastore.NewKey("/message-signer-consensus/"))
-	return &MessageSignerConsensus{
-		MsgSigner: &MessageSigner{
-			wallet: wallet,
-			mpool:  mpool,
-			ds:     ds,
-		},
-		Consensus: consensus,
-	}
-}
-
-func (ms *MessageSignerConsensus) IsLeader(ctx context.Context) bool {
-	return ms.Consensus.IsLeader(ctx)
-}
-
-func (ms *MessageSignerConsensus) RedirectToLeader(ctx context.Context, method string, arg interface{}, ret interface{}) (bool, error) {
-	ok, err := ms.Consensus.RedirectToLeader(method, arg, ret.(*types.SignedMessage))
-	if err != nil {
-		return ok, err
-	}
-	return ok, nil
-}
-
-func (ms *MessageSignerConsensus) SignMessage(
-	ctx context.Context,
-	msg *types.Message,
-	spec *api.MessageSendSpec,
-	cb func(*types.SignedMessage) error) (*types.SignedMessage, error) {
-
-	signedMsg, err := ms.MsgSigner.SignMessage(ctx, msg, spec, cb)
-	if err != nil {
-		return nil, err
-	}
-
-	op := &consensus.ConsensusOp{
-		Nonce:     signedMsg.Message.Nonce,
-		Uuid:      spec.MsgUuid,
-		Addr:      signedMsg.Message.From,
-		SignedMsg: signedMsg,
-	}
-	err = ms.Consensus.Commit(ctx, op)
-	if err != nil {
-		return nil, err
-	}
-
-	return signedMsg, nil
-}
-
-func (ms *MessageSignerConsensus) GetSignedMessage(ctx context.Context, uuid uuid.UUID) (*types.SignedMessage, error) {
-	cstate, err := ms.Consensus.State(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	//cstate := state.(Consensus.RaftState)
-	msg, ok := cstate.MsgUuids[uuid]
-	if !ok {
-		return nil, xerrors.Errorf("Msg with Uuid %s not available", uuid)
-	}
-	return msg, nil
-}
-
-func (ms *MessageSignerConsensus) GetRaftState(ctx context.Context) (*consensus.RaftState, error) {
-	return ms.Consensus.State(ctx)
-}
-
-func (ms *MessageSignerConsensus) Leader(ctx context.Context) (peer.ID, error) {
-	return ms.Consensus.Leader(ctx)
-}

chain/state/statetree.go

@@ -438,7 +438,8 @@ func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
 		return cid.Undef, xerrors.Errorf("tried to flush state tree with snapshots on the stack")
 	}
 
-	for addr, sto := range st.snaps.layers[0].actors {
+	for addr, stoTmp := range st.snaps.layers[0].actors {
+		sto := stoTmp
 		if sto.Delete {
 			if err := st.root.Delete(abi.AddrKey(addr)); err != nil {
 				return cid.Undef, err

chain/sync.go

@@ -886,6 +886,35 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know
 		}
 	}
 
+	incomingParentsTsk := incoming.Parents()
+	commonParent := false
+	for _, incomingParent := range incomingParentsTsk.Cids() {
+		if known.Contains(incomingParent) {
+			commonParent = true
+		}
+	}
+
+	if commonParent {
+		// known contains at least one of incoming's Parents => the common ancestor is known's Parents (incoming's Grandparents)
+		// in this case, we need to return {incoming.Parents()}
+		incomingParents, err := syncer.store.LoadTipSet(ctx, incomingParentsTsk)
+		if err != nil {
+			// fallback onto the network
+			tips, err := syncer.Exchange.GetBlocks(ctx, incoming.Parents(), 1)
+			if err != nil {
+				return nil, xerrors.Errorf("failed to fetch incomingParents from the network: %w", err)
+			}
+
+			if len(tips) == 0 {
+				return nil, xerrors.Errorf("network didn't return any tipsets")
+			}
+
+			incomingParents = tips[0]
+		}
+
+		return []*types.TipSet{incomingParents}, nil
+	}
+
 	// TODO: Does this mean we always ask for ForkLengthThreshold blocks from the network, even if we just need, like, 2? Yes.
 	// Would it not be better to ask in smaller chunks, given that an ~ForkLengthThreshold is very rare?
 	tips, err := syncer.Exchange.GetBlocks(ctx, incoming.Parents(), int(build.ForkLengthThreshold))

chain/types/ethtypes/eth_types.go

@@ -229,13 +229,25 @@ type EthCall struct {
 }
 
 func (c *EthCall) UnmarshalJSON(b []byte) error {
-	type TempEthCall EthCall
-	var params TempEthCall
+	type EthCallRaw EthCall // Avoid a recursive call.
+	type EthCallDecode struct {
+		// The field should be "input" by spec, but many clients use "data" so we support
+		// both, but prefer "input".
+		Input *EthBytes `json:"input"`
+		EthCallRaw
+	}
 
+	var params EthCallDecode
 	if err := json.Unmarshal(b, &params); err != nil {
 		return err
 	}
-	*c = EthCall(params)
+
+	// If input is specified, prefer it.
+	if params.Input != nil {
+		params.Data = *params.Input
+	}
+
+	*c = EthCall(params.EthCallRaw)
 	return nil
 }
 

chain/types/ethtypes/eth_types_test.go

@@ -194,11 +194,40 @@ func TestMaskedIDInF4(t *testing.T) {
 }
 
 func TestUnmarshalEthCall(t *testing.T) {
-	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":""}`
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFF"}`
 
 	var c EthCall
 	err := c.UnmarshalJSON([]byte(data))
 	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInput(t *testing.T) {
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","input":"0xFF"}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInputAndData(t *testing.T) {
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":"0xFF"}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{0xff}, c.Data)
+}
+
+func TestUnmarshalEthCallInputAndDataEmpty(t *testing.T) {
+	// Even if the input is empty, it should be used when specified.
+	data := `{"from":"0x4D6D86b31a112a05A473c4aE84afaF873f632325","to":"0xFe01CC39f5Ae8553D6914DBb9dC27D219fa22D7f","gas":"0x5","gasPrice":"0x6","value":"0x123","data":"0xFE","input":""}`
+
+	var c EthCall
+	err := c.UnmarshalJSON([]byte(data))
+	require.Nil(t, err)
+	require.EqualValues(t, []byte{}, c.Data)
 }
 
 func TestUnmarshalEthBytes(t *testing.T) {

chain/types/fil.go

@@ -12,6 +12,9 @@ import (
 type FIL BigInt
 
 func (f FIL) String() string {
+	if f.Int == nil {
+		return "0 FIL"
+	}
 	return f.Unitless() + " FIL"
 }
 

chain/vectors/gen/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	crand "crypto/rand"
 	"encoding/json"
 	"fmt"
 	"math/rand"
@@ -145,7 +146,10 @@ func MakeUnsignedMessageVectors() []vectors.UnsignedMessageVector {
 		}
 
 		params := make([]byte, 32)
-		rand.Read(params)
+		_, err = crand.Read(params)
+		if err != nil {
+			panic(err)
+		}
 
 		msg := &types.Message{
 			To:         to,

cli/helper.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -8,7 +9,6 @@ import (
 	"syscall"
 
 	ufcli "github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
 )
 
 type PrintHelpErr struct {
@@ -52,7 +52,7 @@ func RunApp(app *ufcli.App) {
 			fmt.Fprintf(os.Stderr, "ERROR: %s\n\n", err) // nolint:errcheck
 		}
 		var phe *PrintHelpErr
-		if xerrors.As(err, &phe) {
+		if errors.As(err, &phe) {
 			_ = ufcli.ShowCommandHelp(phe.Ctx, phe.Ctx.Command.Name)
 		}
 		os.Exit(1)

cli/net.go

@@ -847,7 +847,8 @@ var NetStatCmd = &cli.Command{
 			})
 
 			for _, stat := range stats {
-				printScope(&stat.stat, name+stat.name)
+				tmp := stat.stat
+				printScope(&tmp, name+stat.name)
 			}
 
 		}

cli/util/api.go

@@ -119,7 +119,7 @@ func GetAPIInfoMulti(ctx *cli.Context, t repo.RepoType) ([]APIInfo, error) {
 		}
 	}
 
-	return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v", t.Type())
+	return []APIInfo{}, fmt.Errorf("could not determine API endpoint for node type: %v. Try setting environment variable: %s", t.Type(), primaryEnv)
 }
 
 func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) {
@@ -164,6 +164,28 @@ func GetRawAPIMulti(ctx *cli.Context, t repo.RepoType, version string) ([]HttpHe
 	return httpHeads, nil
 }
 
+func GetRawAPIMultiV2(ctx *cli.Context, ainfoCfg []string, version string) ([]HttpHead, error) {
+	var httpHeads []HttpHead
+
+	if len(ainfoCfg) == 0 {
+		return httpHeads, xerrors.Errorf("could not get API info: none configured. \nConsider getting base.toml with './lotus-provider config get base >/tmp/base.toml' \nthen adding   \n[APIs] \n ChainApiInfo = [\" result_from lotus auth api-info --perm=admin \"]\n  and updating it with './lotus-provider config set /tmp/base.toml'")
+	}
+	for _, i := range ainfoCfg {
+		ainfo := ParseApiInfo(i)
+		addr, err := ainfo.DialArgs(version)
+		if err != nil {
+			return httpHeads, xerrors.Errorf("could not get DialArgs: %w", err)
+		}
+		httpHeads = append(httpHeads, HttpHead{addr: addr, header: ainfo.AuthHeader()})
+	}
+
+	if IsVeryVerbose {
+		_, _ = fmt.Fprintf(ctx.App.Writer, "using raw API %s endpoint: %s\n", version, httpHeads[0].addr)
+	}
+
+	return httpHeads, nil
+}
+
 func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) {
 	heads, err := GetRawAPIMulti(ctx, t, version)
 	if err != nil {
@@ -393,6 +415,68 @@ func GetFullNodeAPIV1(ctx *cli.Context, opts ...GetFullNodeOption) (v1api.FullNo
 	return &v1API, finalCloser, nil
 }
 
+func GetFullNodeAPIV1LotusProvider(ctx *cli.Context, ainfoCfg []string, opts ...GetFullNodeOption) (v1api.FullNode, jsonrpc.ClientCloser, error) {
+	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
+		return tn.(v1api.FullNode), func() {}, nil
+	}
+
+	var options GetFullNodeOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	var rpcOpts []jsonrpc.Option
+	if options.ethSubHandler != nil {
+		rpcOpts = append(rpcOpts, jsonrpc.WithClientHandler("Filecoin", options.ethSubHandler), jsonrpc.WithClientHandlerAlias("eth_subscription", "Filecoin.EthSubscription"))
+	}
+
+	heads, err := GetRawAPIMultiV2(ctx, ainfoCfg, "v1")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if IsVeryVerbose {
+		_, _ = fmt.Fprintln(ctx.App.Writer, "using full node API v1 endpoint:", heads[0].addr)
+	}
+
+	var fullNodes []api.FullNode
+	var closers []jsonrpc.ClientCloser
+
+	for _, head := range heads {
+		v1api, closer, err := client.NewFullNodeRPCV1(ctx.Context, head.addr, head.header, rpcOpts...)
+		if err != nil {
+			log.Warnf("Not able to establish connection to node with addr: %s, Reason: %s", head.addr, err.Error())
+			continue
+		}
+		fullNodes = append(fullNodes, v1api)
+		closers = append(closers, closer)
+	}
+
+	// When running in cluster mode and trying to establish connections to multiple nodes, fail
+	// if less than 2 lotus nodes are actually running
+	if len(heads) > 1 && len(fullNodes) < 2 {
+		return nil, nil, xerrors.Errorf("Not able to establish connection to more than a single node")
+	}
+
+	finalCloser := func() {
+		for _, c := range closers {
+			c()
+		}
+	}
+
+	var v1API api.FullNodeStruct
+	FullNodeProxy(fullNodes, &v1API)
+
+	v, err := v1API.Version(ctx.Context)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !v.APIVersion.EqMajorMinor(api.FullAPIVersion1) {
+		return nil, nil, xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", api.FullAPIVersion1, v.APIVersion)
+	}
+	return &v1API, finalCloser, nil
+}
+
 type GetStorageMinerOptions struct {
 	PreferHttp bool
 }

cmd/lotus-bench/main.go

@@ -3,10 +3,10 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	"math/big"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"sync"
@@ -547,7 +547,10 @@ var sealBenchCmd = &cli.Command{
 		}
 
 		var challenge [32]byte
-		rand.Read(challenge[:])
+		_, err = rand.Read(challenge[:])
+		if err != nil {
+			return err
+		}
 
 		beforePost := time.Now()
 
@@ -777,9 +780,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 		start := time.Now()
 		log.Infof("[%d] Writing piece into sector...", i)
 
-		r := rand.New(rand.NewSource(100 + int64(i)))
-
-		pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r)
+		pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), rand.Reader)
 		if err != nil {
 			return nil, nil, err
 		}

cmd/lotus-miner/init.go

@@ -120,6 +120,11 @@ var initCmd = &cli.Command{
 			Name:  "from",
 			Usage: "select which address to send actor creation message from",
 		},
+		&cli.Uint64Flag{
+			Name:  "confidence",
+			Usage: "number of block confirmations to wait for",
+			Value: build.MessageConfidence,
+		},
 	},
 	Subcommands: []*cli.Command{
 		restoreCmd,
@@ -146,6 +151,8 @@ var initCmd = &cli.Command{
 			return xerrors.Errorf("failed to parse gas-price flag: %s", err)
 		}
 
+		confidence := cctx.Uint64("confidence")
+
 		symlink := cctx.Bool("symlink-imported-sectors")
 		if symlink {
 			log.Info("will attempt to symlink to imported sectors")
@@ -265,7 +272,7 @@ var initCmd = &cli.Command{
 			}
 		}
 
-		if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice); err != nil {
+		if err := storageMinerInit(ctx, cctx, api, r, ssize, gasPrice, confidence); err != nil {
 			log.Errorf("Failed to initialize lotus-miner: %+v", err)
 			path, err := homedir.Expand(repoPath)
 			if err != nil {
@@ -414,7 +421,7 @@ func findMarketDealID(ctx context.Context, api v1api.FullNode, deal markettypes.
 	return 0, xerrors.New("deal not found")
 }
 
-func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt) error {
+func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode, r repo.Repo, ssize abi.SectorSize, gasPrice types.BigInt, confidence uint64) error {
 	lr, err := r.Lock(repo.StorageMiner)
 	if err != nil {
 		return err
@@ -463,7 +470,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix))
 			smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
 
-			si := paths.NewIndex(nil)
+			si := paths.NewMemIndex(nil)
 
 			lstor, err := paths.NewLocal(ctx, lr, si, nil)
 			if err != nil {
@@ -501,7 +508,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 					return xerrors.Errorf("failed to start up genesis miner: %w", err)
 				}
 
-				cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice)
+				cerr := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence)
 
 				if err := m.Stop(ctx); err != nil {
 					log.Error("failed to shut down miner: ", err)
@@ -541,13 +548,13 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			}
 		}
 
-		if err := configureStorageMiner(ctx, api, a, peerid, gasPrice); err != nil {
+		if err := configureStorageMiner(ctx, api, a, peerid, gasPrice, confidence); err != nil {
 			return xerrors.Errorf("failed to configure miner: %w", err)
 		}
 
 		addr = a
 	} else {
-		a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, cctx)
+		a, err := createStorageMiner(ctx, api, ssize, peerid, gasPrice, confidence, cctx)
 		if err != nil {
 			return xerrors.Errorf("creating miner failed: %w", err)
 		}
@@ -589,7 +596,7 @@ func makeHostKey(lr repo.LockedRepo) (crypto.PrivKey, error) {
 	return pk, nil
 }
 
-func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt) error {
+func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address.Address, peerid peer.ID, gasPrice types.BigInt, confidence uint64) error {
 	mi, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK)
 	if err != nil {
 		return xerrors.Errorf("getWorkerAddr returned bad address: %w", err)
@@ -615,7 +622,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address
 	}
 
 	log.Info("Waiting for message: ", smsg.Cid())
-	ret, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+	ret, err := api.StateWaitMsg(ctx, smsg.Cid(), confidence, lapi.LookbackNoLimit, true)
 	if err != nil {
 		return err
 	}
@@ -627,7 +634,7 @@ func configureStorageMiner(ctx context.Context, api v1api.FullNode, addr address
 	return nil
 }
 
-func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, cctx *cli.Context) (address.Address, error) {
+func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.SectorSize, peerid peer.ID, gasPrice types.BigInt, confidence uint64, cctx *cli.Context) (address.Address, error) {
 	var err error
 	var owner address.Address
 	if cctx.String("owner") != "" {
@@ -679,7 +686,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 		log.Infof("Initializing worker account %s, message: %s", worker, signed.Cid())
 		log.Infof("Waiting for confirmation")
 
-		mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+		mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 		if err != nil {
 			return address.Undef, xerrors.Errorf("waiting for worker init: %w", err)
 		}
@@ -703,7 +710,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 		log.Infof("Initializing owner account %s, message: %s", worker, signed.Cid())
 		log.Infof("Waiting for confirmation")
 
-		mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+		mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 		if err != nil {
 			return address.Undef, xerrors.Errorf("waiting for owner init: %w", err)
 		}
@@ -752,7 +759,7 @@ func createStorageMiner(ctx context.Context, api v1api.FullNode, ssize abi.Secto
 	log.Infof("Pushed CreateMiner message: %s", signed.Cid())
 	log.Infof("Waiting for confirmation")
 
-	mw, err := api.StateWaitMsg(ctx, signed.Cid(), build.MessageConfidence, lapi.LookbackNoLimit, true)
+	mw, err := api.StateWaitMsg(ctx, signed.Cid(), confidence, lapi.LookbackNoLimit, true)
 	if err != nil {
 		return address.Undef, xerrors.Errorf("waiting for createMiner message: %w", err)
 	}

cmd/lotus-miner/init_restore.go

@@ -80,8 +80,7 @@ var restoreCmd = &cli.Command{
 			}
 
 			log.Info("Configuring miner actor")
-
-			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
+			if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil {
 				return err
 			}
 

cmd/lotus-miner/init_service.go

@@ -105,7 +105,7 @@ var serviceCmd = &cli.Command{
 			if es.Contains(MarketsService) {
 				log.Info("Configuring miner actor")
 
-				if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil {
+				if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero(), cctx.Uint64("confidence")); err != nil {
 					return err
 				}
 			}

cmd/lotus-miner/proving.go

@@ -559,7 +559,8 @@ var provingCheckProvableCmd = &cli.Command{
 		for parIdx, par := range partitions {
 			sectors := make(map[abi.SectorNumber]struct{})
 
-			sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.LiveSectors, types.EmptyTSK)
+			tmp := par.LiveSectors
+			sectorInfos, err := api.StateMinerSectors(ctx, addr, &tmp, types.EmptyTSK)
 			if err != nil {
 				return err
 			}

cmd/lotus-miner/sectors.go

@@ -2290,7 +2290,7 @@ var sectorsCompactPartitionsCmd = &cli.Command{
 		if len(parts) <= 0 {
 			return fmt.Errorf("must include at least one partition to compact")
 		}
-		fmt.Printf("compacting %d paritions\n", len(parts))
+		fmt.Printf("compacting %d partitions\n", len(parts))
 
 		var makeMsgForPartitions func(partitionsBf bitfield.BitField) ([]*types.Message, error)
 		makeMsgForPartitions = func(partitionsBf bitfield.BitField) ([]*types.Message, error) {

cmd/lotus-provider/config.go

@@ -0,0 +1,241 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+)
+
+var configCmd = &cli.Command{
+	Name:  "config",
+	Usage: "Manage node config by layers. The layer 'base' will always be applied. ",
+	Subcommands: []*cli.Command{
+		configDefaultCmd,
+		configSetCmd,
+		configGetCmd,
+		configListCmd,
+		configViewCmd,
+		configRmCmd,
+		configMigrateCmd,
+	},
+}
+
+var configDefaultCmd = &cli.Command{
+	Name:    "default",
+	Aliases: []string{"defaults"},
+	Usage:   "Print default node config",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "no-comment",
+			Usage: "don't comment default values",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		comment := !cctx.Bool("no-comment")
+		cfg, err := getDefaultConfig(comment)
+		if err != nil {
+			return err
+		}
+		fmt.Print(cfg)
+
+		return nil
+	},
+}
+
+func getDefaultConfig(comment bool) (string, error) {
+	c := config.DefaultLotusProvider()
+	cb, err := config.ConfigUpdate(c, nil, config.Commented(comment), config.DefaultKeepUncommented(), config.NoEnv())
+	if err != nil {
+		return "", err
+	}
+	return string(cb), nil
+}
+
+var configSetCmd = &cli.Command{
+	Name:      "set",
+	Aliases:   []string{"add", "update", "create"},
+	Usage:     "Set a config layer or the base by providing a filename or stdin.",
+	ArgsUsage: "a layer's file name",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "title",
+			Usage: "title of the config layer (req'd for stdin)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		args := cctx.Args()
+
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		name := cctx.String("title")
+		var stream io.Reader = os.Stdin
+		if args.Len() != 1 {
+			if cctx.String("title") == "" {
+				return errors.New("must have a title for stdin, or a file name")
+			}
+		} else {
+			stream, err = os.Open(args.First())
+			if err != nil {
+				return fmt.Errorf("cannot open file %s: %w", args.First(), err)
+			}
+			if name == "" {
+				name = strings.Split(path.Base(args.First()), ".")[0]
+			}
+		}
+		bytes, err := io.ReadAll(stream)
+		if err != nil {
+			return fmt.Errorf("cannot read stream/file %w", err)
+		}
+
+		lp := config.DefaultLotusProvider() // ensure it's toml
+		_, err = toml.Decode(string(bytes), lp)
+		if err != nil {
+			return fmt.Errorf("cannot decode file: %w", err)
+		}
+		_ = lp
+
+		err = setConfig(db, name, string(bytes))
+
+		if err != nil {
+			return fmt.Errorf("unable to save config layer: %w", err)
+		}
+
+		fmt.Println("Layer " + name + " created/updated")
+		return nil
+	},
+}
+
+func setConfig(db *harmonydb.DB, name, config string) error {
+	_, err := db.Exec(context.Background(),
+		`INSERT INTO harmony_config (title, config) VALUES ($1, $2) 
+			ON CONFLICT (title) DO UPDATE SET config = excluded.config`, name, config)
+	return err
+}
+
+var configGetCmd = &cli.Command{
+	Name:      "get",
+	Aliases:   []string{"cat", "show"},
+	Usage:     "Get a config layer by name. You may want to pipe the output to a file, or use 'less'",
+	ArgsUsage: "layer name",
+	Action: func(cctx *cli.Context) error {
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return fmt.Errorf("want 1 layer arg, got %d", args.Len())
+		}
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := getConfig(db, args.First())
+		if err != nil {
+			return err
+		}
+		fmt.Println(cfg)
+
+		return nil
+	},
+}
+
+func getConfig(db *harmonydb.DB, layer string) (string, error) {
+	var cfg string
+	err := db.QueryRow(context.Background(), `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&cfg)
+	if err != nil {
+		return "", err
+	}
+	return cfg, nil
+}
+
+var configListCmd = &cli.Command{
+	Name:    "list",
+	Aliases: []string{"ls"},
+	Usage:   "List config layers you can get.",
+	Flags:   []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+		var res []string
+		err = db.Select(context.Background(), &res, `SELECT title FROM harmony_config ORDER BY title`)
+		if err != nil {
+			return fmt.Errorf("unable to read from db: %w", err)
+		}
+		for _, r := range res {
+			fmt.Println(r)
+		}
+
+		return nil
+	},
+}
+
+var configRmCmd = &cli.Command{
+	Name:    "remove",
+	Aliases: []string{"rm", "del", "delete"},
+	Usage:   "Remove a named config layer.",
+	Flags:   []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		args := cctx.Args()
+		if args.Len() != 1 {
+			return errors.New("must have exactly 1 arg for the layer name")
+		}
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+		ct, err := db.Exec(context.Background(), `DELETE FROM harmony_config WHERE title=$1`, args.First())
+		if err != nil {
+			return fmt.Errorf("unable to read from db: %w", err)
+		}
+		if ct == 0 {
+			return fmt.Errorf("no layer named %s", args.First())
+		}
+
+		return nil
+	},
+}
+var configViewCmd = &cli.Command{
+	Name:      "interpret",
+	Aliases:   []string{"view", "stacked", "stack"},
+	Usage:     "Interpret stacked config layers by this version of lotus-provider, with system-generated comments.",
+	ArgsUsage: "a list of layers to be interpreted as the final config",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "layers",
+			Usage:    "comma or space separated list of layers to be interpreted",
+			Value:    cli.NewStringSlice("base"),
+			Required: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+		lp, err := deps.GetConfig(cctx, db)
+		if err != nil {
+			return err
+		}
+		cb, err := config.ConfigUpdate(lp, config.DefaultLotusProvider(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv())
+		if err != nil {
+			return xerrors.Errorf("cannot interpret config: %w", err)
+		}
+		fmt.Println(string(cb))
+		return nil
+	},
+}

cmd/lotus-provider/deps/deps.go

@@ -0,0 +1,282 @@
+// Package deps provides the dependencies for the lotus provider node.
+package deps
+
+import (
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/gbrlsnchs/jwt/v3"
+	ds "github.com/ipfs/go-datastore"
+	dssync "github.com/ipfs/go-datastore/sync"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+	"github.com/filecoin-project/go-statestore"
+
+	"github.com/filecoin-project/lotus/api"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/journal/alerting"
+	"github.com/filecoin-project/lotus/journal/fsjournal"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/lotus/provider"
+	"github.com/filecoin-project/lotus/storage/ctladdr"
+	"github.com/filecoin-project/lotus/storage/paths"
+	"github.com/filecoin-project/lotus/storage/sealer"
+	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
+	"github.com/filecoin-project/lotus/storage/sealer/storiface"
+)
+
+var log = logging.Logger("lotus-provider/deps")
+
+func MakeDB(cctx *cli.Context) (*harmonydb.DB, error) {
+	dbConfig := config.HarmonyDB{
+		Username: cctx.String("db-user"),
+		Password: cctx.String("db-password"),
+		Hosts:    strings.Split(cctx.String("db-host"), ","),
+		Database: cctx.String("db-name"),
+		Port:     cctx.String("db-port"),
+	}
+	return harmonydb.NewFromConfig(dbConfig)
+}
+
+type JwtPayload struct {
+	Allow []auth.Permission
+}
+
+func StorageAuth(apiKey string) (sealer.StorageAuth, error) {
+	if apiKey == "" {
+		return nil, xerrors.Errorf("no api key provided")
+	}
+
+	rawKey, err := base64.StdEncoding.DecodeString(apiKey)
+	if err != nil {
+		return nil, xerrors.Errorf("decoding api key: %w", err)
+	}
+
+	key := jwt.NewHS256(rawKey)
+
+	p := JwtPayload{
+		Allow: []auth.Permission{"admin"},
+	}
+
+	token, err := jwt.Sign(&p, key)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := http.Header{}
+	headers.Add("Authorization", "Bearer "+string(token))
+	return sealer.StorageAuth(headers), nil
+}
+
+func GetDeps(ctx context.Context, cctx *cli.Context) (*Deps, error) {
+	var deps Deps
+	return &deps, deps.PopulateRemainingDeps(ctx, cctx, true)
+}
+
+type Deps struct {
+	Cfg        *config.LotusProviderConfig
+	DB         *harmonydb.DB
+	Full       api.FullNode
+	Verif      storiface.Verifier
+	LW         *sealer.LocalWorker
+	As         *ctladdr.AddressSelector
+	Maddrs     []dtypes.MinerAddress
+	Stor       *paths.Remote
+	Si         *paths.DBIndex
+	LocalStore *paths.Local
+	ListenAddr string
+}
+
+const (
+	FlagRepoPath = "repo-path"
+)
+
+func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context, makeRepo bool) error {
+	var err error
+	if makeRepo {
+		// Open repo
+		repoPath := cctx.String(FlagRepoPath)
+		fmt.Println("repopath", repoPath)
+		r, err := repo.NewFS(repoPath)
+		if err != nil {
+			return err
+		}
+
+		ok, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !ok {
+			if err := r.Init(repo.Provider); err != nil {
+				return err
+			}
+		}
+	}
+
+	if deps.Cfg == nil {
+		deps.DB, err = MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	if deps.Cfg == nil {
+		// The config feeds into task runners & their helpers
+		deps.Cfg, err = GetConfig(cctx, deps.DB)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Debugw("config", "config", deps.Cfg)
+
+	if deps.Verif == nil {
+		deps.Verif = ffiwrapper.ProofVerifier
+	}
+
+	if deps.As == nil {
+		deps.As, err = provider.AddressSelector(&deps.Cfg.Addresses)()
+		if err != nil {
+			return err
+		}
+	}
+
+	if deps.Si == nil {
+		de, err := journal.ParseDisabledEvents(deps.Cfg.Journal.DisabledEvents)
+		if err != nil {
+			return err
+		}
+		j, err := fsjournal.OpenFSJournalPath(cctx.String("journal"), de)
+		if err != nil {
+			return err
+		}
+		go func() {
+			<-ctx.Done()
+			_ = j.Close()
+		}()
+
+		al := alerting.NewAlertingSystem(j)
+		deps.Si = paths.NewDBIndex(al, deps.DB)
+	}
+
+	if deps.Full == nil {
+		var fullCloser func()
+		cfgApiInfo := deps.Cfg.Apis.ChainApiInfo
+		if v := os.Getenv("FULLNODE_API_INFO"); v != "" {
+			cfgApiInfo = []string{v}
+		}
+		deps.Full, fullCloser, err = cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfgApiInfo)
+		if err != nil {
+			return err
+		}
+
+		go func() {
+			<-ctx.Done()
+			fullCloser()
+		}()
+	}
+
+	bls := &paths.BasicLocalStorage{
+		PathToJSON: cctx.String("storage-json"),
+	}
+
+	if deps.ListenAddr == "" {
+		listenAddr := cctx.String("listen")
+		const unspecifiedAddress = "0.0.0.0"
+		addressSlice := strings.Split(listenAddr, ":")
+		if ip := net.ParseIP(addressSlice[0]); ip != nil {
+			if ip.String() == unspecifiedAddress {
+				rip, err := deps.DB.GetRoutableIP()
+				if err != nil {
+					return err
+				}
+				deps.ListenAddr = rip + ":" + addressSlice[1]
+			}
+		}
+	}
+	if deps.LocalStore == nil {
+		deps.LocalStore, err = paths.NewLocal(ctx, bls, deps.Si, []string{"http://" + deps.ListenAddr + "/remote"})
+		if err != nil {
+			return err
+		}
+	}
+
+	sa, err := StorageAuth(deps.Cfg.Apis.StorageRPCSecret)
+	if err != nil {
+		return xerrors.Errorf(`'%w' while parsing the config toml's 
+	[Apis]
+	StorageRPCSecret=%v
+Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, err, deps.Cfg.Apis.StorageRPCSecret)
+	}
+	if deps.Stor == nil {
+		deps.Stor = paths.NewRemote(deps.LocalStore, deps.Si, http.Header(sa), 10, &paths.DefaultPartialFileHandler{})
+	}
+	if deps.LW == nil {
+		wstates := statestore.New(dssync.MutexWrap(ds.NewMapDatastore()))
+
+		// todo localWorker isn't the abstraction layer we want to use here, we probably want to go straight to ffiwrapper
+		//  maybe with a lotus-provider specific abstraction. LocalWorker does persistent call tracking which we probably
+		//  don't need (ehh.. maybe we do, the async callback system may actually work decently well with harmonytask)
+		deps.LW = sealer.NewLocalWorker(sealer.WorkerConfig{}, deps.Stor, deps.LocalStore, deps.Si, nil, wstates)
+	}
+	if len(deps.Maddrs) == 0 {
+		for _, s := range deps.Cfg.Addresses.MinerAddresses {
+			addr, err := address.NewFromString(s)
+			if err != nil {
+				return err
+			}
+			deps.Maddrs = append(deps.Maddrs, dtypes.MinerAddress(addr))
+		}
+	}
+	fmt.Println("last line of populate")
+	return nil
+}
+
+func GetConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig, error) {
+	lp := config.DefaultLotusProvider()
+	have := []string{}
+	layers := cctx.StringSlice("layers")
+	for _, layer := range layers {
+		text := ""
+		err := db.QueryRow(cctx.Context, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
+		if err != nil {
+			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
+				return nil, fmt.Errorf("missing layer '%s' ", layer)
+			}
+			if layer == "base" {
+				return nil, errors.New(`lotus-provider defaults to a layer named 'base'. 
+				Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`)
+			}
+			return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
+		}
+		meta, err := toml.Decode(text, &lp)
+		if err != nil {
+			return nil, fmt.Errorf("could not read layer, bad toml %s: %w", layer, err)
+		}
+		for _, k := range meta.Keys() {
+			have = append(have, strings.Join(k, " "))
+		}
+		log.Infow("Using layer", "layer", layer, "config", lp)
+	}
+	_ = have // FUTURE: verify that required fields are here.
+	// If config includes 3rd-party config, consider JSONSchema as a way that
+	// 3rd-parties can dynamically include config requirements and we can
+	// validate the config. Because of layering, we must validate @ startup.
+	return lp, nil
+}

cmd/lotus-provider/main.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"runtime/pprof"
+	"syscall"
+
+	"github.com/fatih/color"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/lotuslog"
+	"github.com/filecoin-project/lotus/lib/tracing"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var log = logging.Logger("main")
+
+func SetupCloseHandler() {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		fmt.Println("\r- Ctrl+C pressed in Terminal")
+		_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		panic(1)
+	}()
+}
+
+func main() {
+	SetupCloseHandler()
+
+	lotuslog.SetupLogLevels()
+
+	local := []*cli.Command{
+		//initCmd,
+		runCmd,
+		stopCmd,
+		configCmd,
+		testCmd,
+		webCmd,
+		//backupCmd,
+		//lcli.WithCategory("chain", actorCmd),
+		//lcli.WithCategory("storage", sectorsCmd),
+		//lcli.WithCategory("storage", provingCmd),
+		//lcli.WithCategory("storage", storageCmd),
+		//lcli.WithCategory("storage", sealingCmd),
+	}
+
+	jaeger := tracing.SetupJaegerTracing("lotus")
+	defer func() {
+		if jaeger != nil {
+			_ = jaeger.ForceFlush(context.Background())
+		}
+	}()
+
+	for _, cmd := range local {
+		cmd := cmd
+		originBefore := cmd.Before
+		cmd.Before = func(cctx *cli.Context) error {
+			if jaeger != nil {
+				_ = jaeger.Shutdown(cctx.Context)
+			}
+			jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name)
+
+			if cctx.IsSet("color") {
+				color.NoColor = !cctx.Bool("color")
+			}
+
+			if originBefore != nil {
+				return originBefore(cctx)
+			}
+
+			return nil
+		}
+	}
+
+	app := &cli.App{
+		Name:                 "lotus-provider",
+		Usage:                "Filecoin decentralized storage network provider",
+		Version:              build.UserVersion(),
+		EnableBashCompletion: true,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				// examined in the Before above
+				Name:        "color",
+				Usage:       "use color in display output",
+				DefaultText: "depends on output being a TTY",
+			},
+			&cli.StringFlag{
+				Name:    "panic-reports",
+				EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"},
+				Hidden:  true,
+				Value:   "~/.lotusprovider", // should follow --repo default
+			},
+			&cli.StringFlag{
+				Name:    "db-host",
+				EnvVars: []string{"LOTUS_DB_HOST"},
+				Usage:   "Command separated list of hostnames for yugabyte cluster",
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-name",
+				EnvVars: []string{"LOTUS_DB_NAME", "LOTUS_HARMONYDB_HOSTS"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-user",
+				EnvVars: []string{"LOTUS_DB_USER", "LOTUS_HARMONYDB_USERNAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-password",
+				EnvVars: []string{"LOTUS_DB_PASSWORD", "LOTUS_HARMONYDB_PASSWORD"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-port",
+				EnvVars: []string{"LOTUS_DB_PORT", "LOTUS_HARMONYDB_PORT"},
+				Hidden:  true,
+				Value:   "5433",
+			},
+			&cli.StringFlag{
+				Name:    "layers",
+				EnvVars: []string{"LOTUS_LAYERS", "LOTUS_CONFIG_LAYERS"},
+				Value:   "base",
+			},
+			&cli.StringFlag{
+				Name:    deps.FlagRepoPath,
+				EnvVars: []string{"LOTUS_REPO_PATH"},
+				Value:   "~/.lotusprovider",
+			},
+			cliutil.FlagVeryVerbose,
+		},
+		Commands: append(local, lcli.CommonCommands...),
+		Before: func(c *cli.Context) error {
+			return nil
+		},
+		After: func(c *cli.Context) error {
+			if r := recover(); r != nil {
+				p, err := homedir.Expand(c.String(FlagMinerRepo))
+				if err != nil {
+					log.Errorw("could not expand repo path for panic report", "error", err)
+					panic(r)
+				}
+
+				// Generate report in LOTUS_PATH and re-raise panic
+				build.GeneratePanicReport(c.String("panic-reports"), p, c.App.Name)
+				panic(r)
+			}
+			return nil
+		},
+	}
+	app.Setup()
+	app.Metadata["repoType"] = repo.Provider
+	lcli.RunApp(app)
+}

cmd/lotus-provider/migrate.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/fatih/color"
+	"github.com/ipfs/go-datastore"
+	"github.com/samber/lo"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var configMigrateCmd = &cli.Command{
+	Name:        "from-miner",
+	Usage:       "Express a database config (for lotus-provider) from an existing miner.",
+	Description: "Express a database config (for lotus-provider) from an existing miner.",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    FlagMinerRepo,
+			Aliases: []string{FlagMinerRepoDeprecation},
+			EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"},
+			Value:   "~/.lotusminer",
+			Usage:   fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation),
+		},
+		&cli.StringFlag{
+			Name:    "repo",
+			EnvVars: []string{"LOTUS_PATH"},
+			Hidden:  true,
+			Value:   "~/.lotus",
+		},
+		&cli.StringFlag{
+			Name:    "to-layer",
+			Aliases: []string{"t"},
+			Usage:   "The layer name for this data push. 'base' is recommended for single-miner setup.",
+		},
+		&cli.BoolFlag{
+			Name:    "overwrite",
+			Aliases: []string{"o"},
+			Usage:   "Use this with --to-layer to replace an existing layer",
+		},
+	},
+	Action: fromMiner,
+}
+
+const (
+	FlagMinerRepo = "miner-repo"
+)
+
+const FlagMinerRepoDeprecation = "storagerepo"
+
+func fromMiner(cctx *cli.Context) (err error) {
+	ctx := context.Background()
+	cliCommandColor := color.New(color.FgHiBlue).SprintFunc()
+	configColor := color.New(color.FgHiGreen).SprintFunc()
+
+	r, err := repo.NewFS(cctx.String(FlagMinerRepo))
+	if err != nil {
+		return err
+	}
+
+	ok, err := r.Exists()
+	if err != nil {
+		return err
+	}
+
+	if !ok {
+		return fmt.Errorf("repo not initialized")
+	}
+
+	lr, err := r.LockRO(repo.StorageMiner)
+	if err != nil {
+		return fmt.Errorf("locking repo: %w", err)
+	}
+	defer func() { _ = lr.Close() }()
+
+	cfgNode, err := lr.Config()
+	if err != nil {
+		return fmt.Errorf("getting node config: %w", err)
+	}
+	smCfg := cfgNode.(*config.StorageMiner)
+
+	db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB)
+	if err != nil {
+		return fmt.Errorf("could not reach the database. Ensure the Miner config toml's HarmonyDB entry"+
+			" is setup to reach Yugabyte correctly: %w", err)
+	}
+
+	var titles []string
+	err = db.Select(ctx, &titles, `SELECT title FROM harmony_config WHERE LENGTH(config) > 0`)
+	if err != nil {
+		return fmt.Errorf("miner cannot reach the db. Ensure the config toml's HarmonyDB entry"+
+			" is setup to reach Yugabyte correctly: %s", err.Error())
+	}
+	name := cctx.String("to-layer")
+	if name == "" {
+		name = fmt.Sprintf("mig%d", len(titles))
+	} else {
+		if lo.Contains(titles, name) && !cctx.Bool("overwrite") {
+			return errors.New("the overwrite flag is needed to replace existing layer: " + name)
+		}
+	}
+	msg := "Layer " + configColor(name) + ` created. `
+
+	// Copy over identical settings:
+
+	buf, err := os.ReadFile(path.Join(lr.Path(), "config.toml"))
+	if err != nil {
+		return fmt.Errorf("could not read config.toml: %w", err)
+	}
+	var lpCfg config.LotusProviderConfig
+	_, err = toml.Decode(string(buf), &lpCfg)
+	if err != nil {
+		return fmt.Errorf("could not decode toml: %w", err)
+	}
+
+	// Populate Miner Address
+	mmeta, err := lr.Datastore(ctx, "/metadata")
+	if err != nil {
+		return xerrors.Errorf("opening miner metadata datastore: %w", err)
+	}
+	defer func() {
+		_ = mmeta.Close()
+	}()
+
+	maddrBytes, err := mmeta.Get(ctx, datastore.NewKey("miner-address"))
+	if err != nil {
+		return xerrors.Errorf("getting miner address datastore entry: %w", err)
+	}
+
+	addr, err := address.NewFromBytes(maddrBytes)
+	if err != nil {
+		return xerrors.Errorf("parsing miner actor address: %w", err)
+	}
+
+	lpCfg.Addresses.MinerAddresses = []string{addr.String()}
+
+	ks, err := lr.KeyStore()
+	if err != nil {
+		return xerrors.Errorf("keystore err: %w", err)
+	}
+	js, err := ks.Get(modules.JWTSecretName)
+	if err != nil {
+		return xerrors.Errorf("error getting JWTSecretName: %w", err)
+	}
+
+	lpCfg.Apis.StorageRPCSecret = base64.StdEncoding.EncodeToString(js.PrivateKey)
+
+	// Populate API Key
+	_, header, err := cliutil.GetRawAPI(cctx, repo.FullNode, "v0")
+	if err != nil {
+		return fmt.Errorf("cannot read API: %w", err)
+	}
+
+	ainfo, err := cliutil.GetAPIInfo(&cli.Context{}, repo.FullNode)
+	if err != nil {
+		return xerrors.Errorf(`could not get API info for FullNode: %w
+		Set the environment variable to the value of "lotus auth api-info --perm=admin"`, err)
+	}
+	lpCfg.Apis.ChainApiInfo = []string{header.Get("Authorization")[7:] + ":" + ainfo.Addr}
+
+	// Enable WindowPoSt
+	lpCfg.Subsystems.EnableWindowPost = true
+	msg += "\nBefore running lotus-provider, ensure any miner/worker answering of WindowPost is disabled by " +
+		"(on Miner) " + configColor("DisableBuiltinWindowPoSt=true") + " and (on Workers) not enabling windowpost on CLI or via " +
+		"environment variable " + configColor("LOTUS_WORKER_WINDOWPOST") + "."
+
+	// Express as configTOML
+	configTOML := &bytes.Buffer{}
+	if err = toml.NewEncoder(configTOML).Encode(lpCfg); err != nil {
+		return err
+	}
+
+	if !lo.Contains(titles, "base") {
+		cfg, err := getDefaultConfig(true)
+		if err != nil {
+			return xerrors.Errorf("Cannot get default config: %w", err)
+		}
+		_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', $1)", cfg)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	if cctx.Bool("overwrite") {
+		i, err := db.Exec(ctx, "DELETE FROM harmony_config WHERE title=$1", name)
+		if i != 0 {
+			fmt.Println("Overwriting existing layer")
+		}
+		if err != nil {
+			fmt.Println("Got error while deleting existing layer: " + err.Error())
+		}
+	}
+
+	_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", name, configTOML.String())
+	if err != nil {
+		return err
+	}
+
+	dbSettings := ""
+	def := config.DefaultStorageMiner().HarmonyDB
+	if def.Hosts[0] != smCfg.HarmonyDB.Hosts[0] {
+		dbSettings += ` --db-host="` + strings.Join(smCfg.HarmonyDB.Hosts, ",") + `"`
+	}
+	if def.Port != smCfg.HarmonyDB.Port {
+		dbSettings += " --db-port=" + smCfg.HarmonyDB.Port
+	}
+	if def.Username != smCfg.HarmonyDB.Username {
+		dbSettings += ` --db-user="` + smCfg.HarmonyDB.Username + `"`
+	}
+	if def.Password != smCfg.HarmonyDB.Password {
+		dbSettings += ` --db-password="` + smCfg.HarmonyDB.Password + `"`
+	}
+	if def.Database != smCfg.HarmonyDB.Database {
+		dbSettings += ` --db-name="` + smCfg.HarmonyDB.Database + `"`
+	}
+
+	var layerMaybe string
+	if name != "base" {
+		layerMaybe = "--layer=" + name
+	}
+
+	msg += `
+To work with the config:
+` + cliCommandColor(`lotus-provider `+dbSettings+` config help `)
+	msg += `
+To run Lotus Provider: in its own machine or cgroup without other files, use the command: 
+` + cliCommandColor(`lotus-provider `+dbSettings+` run `+layerMaybe)
+	fmt.Println(msg)
+	return nil
+}

cmd/lotus-provider/proving.go

@@ -0,0 +1,207 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/dline"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/provider"
+)
+
+var testCmd = &cli.Command{
+	Name:  "test",
+	Usage: "Utility functions for testing",
+	Subcommands: []*cli.Command{
+		//provingInfoCmd,
+		wdPostCmd,
+	},
+}
+
+var wdPostCmd = &cli.Command{
+	Name:    "window-post",
+	Aliases: []string{"wd", "windowpost", "wdpost"},
+	Usage:   "Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain.",
+	Subcommands: []*cli.Command{
+		wdPostHereCmd,
+		wdPostTaskCmd,
+	},
+}
+
+// wdPostTaskCmd writes to harmony_task and wdpost_partition_tasks, then waits for the result.
+// It is intended to be used to test the windowpost scheduler.
+// The end of the compute task puts the task_id onto wdpost_proofs, which is read by the submit task.
+// The submit task will not send test tasks to the chain, and instead will write the result to harmony_test.
+// The result is read by this command, and printed to stdout.
+var wdPostTaskCmd = &cli.Command{
+	Name:    "task",
+	Aliases: []string{"scheduled", "schedule", "async", "asynchronous"},
+	Usage:   "Test the windowpost scheduler by running it on the next available lotus-provider. ",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "deadline",
+			Usage: "deadline to compute WindowPoSt for ",
+			Value: 0,
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base",
+			Value: cli.NewStringSlice("base"),
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.Background()
+
+		deps, err := deps.GetDeps(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		ts, err := deps.Full.ChainHead(ctx)
+		if err != nil {
+			return xerrors.Errorf("cannot get chainhead %w", err)
+		}
+		ht := ts.Height()
+
+		addr, err := address.NewFromString(deps.Cfg.Addresses.MinerAddresses[0])
+		if err != nil {
+			return xerrors.Errorf("cannot get miner address %w", err)
+		}
+		maddr, err := address.IDFromAddress(addr)
+		if err != nil {
+			return xerrors.Errorf("cannot get miner id %w", err)
+		}
+		var id int64
+
+		retryDelay := time.Millisecond * 10
+	retryAddTask:
+		_, err = deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
+			err = tx.QueryRow(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123) RETURNING id`).Scan(&id)
+			if err != nil {
+				log.Error("inserting harmony_task: ", err)
+				return false, xerrors.Errorf("inserting harmony_task: %w", err)
+			}
+			_, err = tx.Exec(`INSERT INTO wdpost_partition_tasks 
+			(task_id, sp_id, proving_period_start, deadline_index, partition_index) VALUES ($1, $2, $3, $4, $5)`,
+				id, maddr, ht, cctx.Uint64("deadline"), 0)
+			if err != nil {
+				log.Error("inserting wdpost_partition_tasks: ", err)
+				return false, xerrors.Errorf("inserting wdpost_partition_tasks: %w", err)
+			}
+			_, err = tx.Exec("INSERT INTO harmony_test (task_id) VALUES ($1)", id)
+			if err != nil {
+				return false, xerrors.Errorf("inserting into harmony_tests: %w", err)
+			}
+			return true, nil
+		})
+		if err != nil {
+			if harmonydb.IsErrSerialization(err) {
+				time.Sleep(retryDelay)
+				retryDelay *= 2
+				goto retryAddTask
+			}
+			return xerrors.Errorf("writing SQL transaction: %w", err)
+		}
+		fmt.Printf("Inserted task %v. Waiting for success ", id)
+		var result sql.NullString
+		for {
+			time.Sleep(time.Second)
+			err = deps.DB.QueryRow(ctx, `SELECT result FROM harmony_test WHERE task_id=$1`, id).Scan(&result)
+			if err != nil {
+				return xerrors.Errorf("reading result from harmony_test: %w", err)
+			}
+			if result.Valid {
+				break
+			}
+			fmt.Print(".")
+		}
+		log.Infof("Result: %s", result.String)
+		return nil
+	},
+}
+
+// This command is intended to be used to verify PoSt compute performance.
+// It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.
+// The entire processing happens in this process while you wait. It does not use the scheduler.
+var wdPostHereCmd = &cli.Command{
+	Name:    "here",
+	Aliases: []string{"cli"},
+	Usage:   "Compute WindowPoSt for performance and configuration testing.",
+	Description: `Note: This command is intended to be used to verify PoSt compute performance.
+It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.`,
+	ArgsUsage: "[deadline index]",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "deadline",
+			Usage: "deadline to compute WindowPoSt for ",
+			Value: 0,
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.StringFlag{
+			Name:  "storage-json",
+			Usage: "path to json file containing storage config",
+			Value: "~/.lotus-provider/storage.json",
+		},
+		&cli.Uint64Flag{
+			Name:  "partition",
+			Usage: "partition to compute WindowPoSt for",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+
+		ctx := context.Background()
+		deps, err := deps.GetDeps(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.Cfg.Fees, deps.Cfg.Proving, deps.Full, deps.Verif, deps.LW, nil,
+			deps.As, deps.Maddrs, deps.DB, deps.Stor, deps.Si, deps.Cfg.Subsystems.WindowPostMaxTasks)
+		if err != nil {
+			return err
+		}
+		_, _ = wdPoStSubmitTask, derlareRecoverTask
+
+		if len(deps.Maddrs) == 0 {
+			return errors.New("no miners to compute WindowPoSt for")
+		}
+		head, err := deps.Full.ChainHead(ctx)
+		if err != nil {
+			return xerrors.Errorf("failed to get chain head: %w", err)
+		}
+
+		di := dline.NewInfo(head.Height(), cctx.Uint64("deadline"), 0, 0, 0, 10 /*challenge window*/, 0, 0)
+
+		for _, maddr := range deps.Maddrs {
+			out, err := wdPostTask.DoPartition(ctx, head, address.Address(maddr), di, cctx.Uint64("partition"))
+			if err != nil {
+				fmt.Println("Error computing WindowPoSt for miner", maddr, err)
+				continue
+			}
+			fmt.Println("Computed WindowPoSt for miner", maddr, ":")
+			err = json.NewEncoder(os.Stdout).Encode(out)
+			if err != nil {
+				fmt.Println("Could not encode WindowPoSt output for miner", maddr, err)
+				continue
+			}
+		}
+
+		return nil
+	},
+}

cmd/lotus-provider/rpc/rpc.go

@@ -0,0 +1,155 @@
+// Package rpc provides all direct access to this node.
+package rpc
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/gbrlsnchs/jwt/v3"
+	"github.com/gorilla/mux"
+	logging "github.com/ipfs/go-log/v2"
+	"go.opencensus.io/tag"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-jsonrpc"
+	"github.com/filecoin-project/go-jsonrpc/auth"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/rpcenc"
+	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/metrics/proxy"
+	"github.com/filecoin-project/lotus/provider/lpweb"
+	"github.com/filecoin-project/lotus/storage/paths"
+)
+
+var log = logging.Logger("lp/rpc")
+
+func LotusProviderHandler(
+	authv func(ctx context.Context, token string) ([]auth.Permission, error),
+	remote http.HandlerFunc,
+	a api.LotusProvider,
+	permissioned bool) http.Handler {
+	mux := mux.NewRouter()
+	readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
+	rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt)
+
+	wapi := proxy.MetricedAPI[api.LotusProvider, api.LotusProviderStruct](a)
+	if permissioned {
+		wapi = api.PermissionedAPI[api.LotusProvider, api.LotusProviderStruct](wapi)
+	}
+
+	rpcServer.Register("Filecoin", wapi)
+	rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
+
+	mux.Handle("/rpc/v0", rpcServer)
+	mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler)
+	mux.PathPrefix("/remote").HandlerFunc(remote)
+	mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
+
+	if !permissioned {
+		return mux
+	}
+
+	ah := &auth.Handler{
+		Verify: authv,
+		Next:   mux.ServeHTTP,
+	}
+	return ah
+}
+
+type ProviderAPI struct {
+	*deps.Deps
+	ShutdownChan chan struct{}
+}
+
+func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
+	return api.ProviderAPIVersion0, nil
+}
+
+// Trigger shutdown
+func (p *ProviderAPI) Shutdown(context.Context) error {
+	close(p.ShutdownChan)
+	return nil
+}
+
+func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error {
+	fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}}
+	remoteHandler := func(w http.ResponseWriter, r *http.Request) {
+		if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
+			w.WriteHeader(401)
+			_ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"})
+			return
+		}
+
+		fh.ServeHTTP(w, r)
+	}
+	// local APIs
+	{
+		// debugging
+		mux := mux.NewRouter()
+		mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
+		mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
+	}
+
+	var authVerify func(context.Context, string) ([]auth.Permission, error)
+	{
+		privateKey, err := base64.StdEncoding.DecodeString(dependencies.Cfg.Apis.StorageRPCSecret)
+		if err != nil {
+			return xerrors.Errorf("decoding storage rpc secret: %w", err)
+		}
+		authVerify = func(ctx context.Context, token string) ([]auth.Permission, error) {
+			var payload deps.JwtPayload
+			if _, err := jwt.Verify([]byte(token), jwt.NewHS256(privateKey), &payload); err != nil {
+				return nil, xerrors.Errorf("JWT Verification failed: %w", err)
+			}
+
+			return payload.Allow, nil
+		}
+	}
+	// Serve the RPC.
+	srv := &http.Server{
+		Handler: LotusProviderHandler(
+			authVerify,
+			remoteHandler,
+			&ProviderAPI{dependencies, shutdownChan},
+			true),
+		ReadHeaderTimeout: time.Minute * 3,
+		BaseContext: func(listener net.Listener) context.Context {
+			ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker"))
+			return ctx
+		},
+		Addr: dependencies.ListenAddr,
+	}
+
+	log.Infof("Setting up RPC server at %s", dependencies.ListenAddr)
+	eg := errgroup.Group{}
+	eg.Go(srv.ListenAndServe)
+
+	if dependencies.Cfg.Subsystems.EnableWebGui {
+		web, err := lpweb.GetSrv(ctx, dependencies)
+		if err != nil {
+			return err
+		}
+
+		go func() {
+			<-ctx.Done()
+			log.Warn("Shutting down...")
+			if err := srv.Shutdown(context.TODO()); err != nil {
+				log.Errorf("shutting down RPC server failed: %s", err)
+			}
+			if err := web.Shutdown(context.Background()); err != nil {
+				log.Errorf("shutting down web server failed: %s", err)
+			}
+			log.Warn("Graceful shutdown successful")
+		}()
+		log.Infof("Setting up web server at %s", dependencies.Cfg.Subsystems.GuiAddress)
+		eg.Go(web.ListenAndServe)
+	}
+	return eg.Wait()
+}

cmd/lotus-provider/run.go

@@ -0,0 +1,189 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/tasks"
+	"github.com/filecoin-project/lotus/lib/ulimit"
+	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/node"
+)
+
+type stackTracer interface {
+	StackTrace() errors.StackTrace
+}
+
+var runCmd = &cli.Command{
+	Name:  "run",
+	Usage: "Start a lotus provider process",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "listen",
+			Usage:   "host address and port the worker api will listen on",
+			Value:   "0.0.0.0:12300",
+			EnvVars: []string{"LOTUS_WORKER_LISTEN"},
+		},
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+		&cli.BoolFlag{
+			Name:   "halt-after-init",
+			Usage:  "only run init, then return",
+			Hidden: true,
+		},
+		&cli.BoolFlag{
+			Name:  "manage-fdlimit",
+			Usage: "manage open file limit",
+			Value: true,
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.StringFlag{
+			Name:  "storage-json",
+			Usage: "path to json file containing storage config",
+			Value: "~/.lotus-provider/storage.json",
+		},
+		&cli.StringFlag{
+			Name:  "journal",
+			Usage: "path to journal files",
+			Value: "~/.lotus-provider/",
+		},
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		defer func() {
+			if err != nil {
+				if err, ok := err.(stackTracer); ok {
+					for _, f := range err.StackTrace() {
+						fmt.Printf("%+s:%d\n", f, f)
+					}
+				}
+			}
+		}()
+		if !cctx.Bool("enable-gpu-proving") {
+			err := os.Setenv("BELLMAN_NO_GPU", "true")
+			if err != nil {
+				return err
+			}
+		}
+
+		ctx, _ := tag.New(lcli.DaemonContext(cctx),
+			tag.Insert(metrics.Version, build.BuildVersion),
+			tag.Insert(metrics.Commit, build.CurrentCommit),
+			tag.Insert(metrics.NodeType, "provider"),
+		)
+		shutdownChan := make(chan struct{})
+		{
+			var ctxclose func()
+			ctx, ctxclose = context.WithCancel(ctx)
+			go func() {
+				<-shutdownChan
+				ctxclose()
+			}()
+		}
+		// Register all metric views
+		/*
+			if err := view.Register(
+				metrics.MinerNodeViews...,
+			); err != nil {
+				log.Fatalf("Cannot register the view: %v", err)
+			}
+		*/
+		// Set the metric to one so it is published to the exporter
+		stats.Record(ctx, metrics.LotusInfo.M(1))
+
+		if cctx.Bool("manage-fdlimit") {
+			if _, _, err := ulimit.ManageFdLimit(); err != nil {
+				log.Errorf("setting file descriptor limit: %s", err)
+			}
+		}
+
+		dependencies := &deps.Deps{}
+		err = dependencies.PopulateRemainingDeps(ctx, cctx, true)
+		if err != nil {
+			fmt.Println("err", err)
+			return err
+		}
+		fmt.Println("ef")
+
+		taskEngine, err := tasks.StartTasks(ctx, dependencies)
+		fmt.Println("gh")
+
+		if err != nil {
+			return nil
+		}
+		defer taskEngine.GracefullyTerminate(time.Hour)
+
+		err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown.
+		if err != nil {
+			return err
+		}
+		finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
+		//node.ShutdownHandler{Component: "provider", StopFunc: stop},
+
+		<-finishCh
+		return nil
+	},
+}
+
+var webCmd = &cli.Command{
+	Name:  "web",
+	Usage: "Start lotus provider web interface",
+	Description: `Start an instance of lotus provider web interface. 
+	This creates the 'web' layer if it does not exist, then calls run with that layer.`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "listen",
+			Usage: "Address to listen on",
+			Value: "127.0.0.1:4701",
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base. Web will be added",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.BoolFlag{
+			Name:  "nosync",
+			Usage: "don't check full-node sync status",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		db, err := deps.MakeDB(cctx)
+		if err != nil {
+			return err
+		}
+
+		webtxt, err := getConfig(db, "web")
+		if err != nil || webtxt == "" {
+
+			s := `[Susbystems]
+			EnableWebGui = true
+			`
+			if err = setConfig(db, "web", s); err != nil {
+				return err
+			}
+		}
+		layers := append([]string{"web"}, cctx.StringSlice("layers")...)
+		err = cctx.Set("layers", strings.Join(layers, ","))
+		if err != nil {
+			return err
+		}
+		return runCmd.Action(cctx)
+	},
+}

cmd/lotus-provider/stop.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	_ "net/http/pprof"
+
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var stopCmd = &cli.Command{
+	Name:  "stop",
+	Usage: "Stop a running lotus provider",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		err = api.Shutdown(lcli.ReqContext(cctx))
+		if err != nil {
+			return err
+		}
+
+		return nil
+	},
+}

cmd/lotus-provider/tasks/tasks.go

@@ -0,0 +1,58 @@
+// Package tasks contains tasks that can be run by the lotus-provider command.
+package tasks
+
+import (
+	"context"
+
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/samber/lo"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
+	"github.com/filecoin-project/lotus/provider"
+	"github.com/filecoin-project/lotus/provider/lpmessage"
+	"github.com/filecoin-project/lotus/provider/lpwinning"
+)
+
+var log = logging.Logger("lotus-provider/deps")
+
+func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.TaskEngine, error) {
+	cfg := dependencies.Cfg
+	db := dependencies.DB
+	full := dependencies.Full
+	verif := dependencies.Verif
+	lw := dependencies.LW
+	as := dependencies.As
+	maddrs := dependencies.Maddrs
+	stor := dependencies.Stor
+	si := dependencies.Si
+	var activeTasks []harmonytask.TaskInterface
+
+	sender, sendTask := lpmessage.NewSender(full, full, db)
+	activeTasks = append(activeTasks, sendTask)
+
+	///////////////////////////////////////////////////////////////////////
+	///// Task Selection
+	///////////////////////////////////////////////////////////////////////
+	{
+
+		if cfg.Subsystems.EnableWindowPost {
+			wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, cfg.Fees, cfg.Proving, full, verif, lw, sender,
+				as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks)
+			if err != nil {
+				return nil, err
+			}
+			activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask)
+		}
+
+		if cfg.Subsystems.EnableWinningPost {
+			winPoStTask := lpwinning.NewWinPostTask(cfg.Subsystems.WinningPostMaxTasks, db, lw, verif, full, maddrs)
+			activeTasks = append(activeTasks, winPoStTask)
+		}
+	}
+	log.Infow("This lotus_provider instance handles",
+		"miner_addresses", maddrs,
+		"tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name }))
+
+	return harmonytask.New(db, activeTasks, dependencies.ListenAddr)
+}

cmd/lotus-shed/invariants.go

@@ -3,7 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
-	"io"
+	"os"
+	"path/filepath"
 	"strconv"
 	"time"
 
@@ -21,6 +22,8 @@ import (
 	v9 "github.com/filecoin-project/go-state-types/builtin/v9"
 
 	"github.com/filecoin-project/lotus/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
+	"github.com/filecoin-project/lotus/blockstore/splitstore"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
@@ -73,24 +76,52 @@ var invariantsCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
+		cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
-			return fmt.Errorf("failed to open blockstore: %w", err)
+			return fmt.Errorf("failed to open universal blockstore %w", err)
 		}
 
-		defer func() {
-			if c, ok := bs.(io.Closer); ok {
-				if err := c.Close(); err != nil {
-					log.Warnf("failed to close blockstore: %s", err)
-				}
-			}
-		}()
+		path, err := lkrepo.SplitstorePath()
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(path, "hot.badger")
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+
+		opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly())
+		if err != nil {
+			return err
+		}
+
+		hot, err := badgerbs.Open(opts)
+		if err != nil {
+			return err
+		}
 
 		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
 
+		cfg := &splitstore.Config{
+			MarkSetType:       "map",
+			DiscardColdBlocks: true,
+		}
+		ss, err := splitstore.Open(path, mds, hot, cold, cfg)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := ss.Close(); err != nil {
+				log.Warnf("failed to close blockstore: %s", err)
+
+			}
+		}()
+		bs := ss
+
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 		defer cs.Close() //nolint:errcheck
 

cmd/lotus-shed/miner.go

@@ -553,7 +553,7 @@ var sendInvalidWindowPoStCmd = &cli.Command{
 			return xerrors.Errorf("serializing params: %w", err)
 		}
 
-		fmt.Printf("submitting bad PoST for %d paritions\n", len(partitionIndices))
+		fmt.Printf("submitting bad PoST for %d partitions\n", len(partitionIndices))
 		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
 			From:   minfo.Worker,
 			To:     maddr,

cmd/lotus-shed/terminations.go

@@ -157,7 +157,8 @@ var terminationsCmd = &cli.Command{
 				}
 
 				for _, t := range termParams.Terminations {
-					sectors, err := minerSt.LoadSectors(&t.Sectors)
+					tmp := t.Sectors
+					sectors, err := minerSt.LoadSectors(&tmp)
 					if err != nil {
 						return err
 					}

cmd/lotus-sim/simulation/stages/funding_stage.go

@@ -166,7 +166,8 @@ func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.Block
 		)
 	}()
 
-	for _, actor := range targets {
+	for _, actorTmp := range targets {
+		actor := actorTmp
 		switch {
 		case builtin.IsAccountActor(actor.Code):
 			if _, err := bb.PushMessage(&types.Message{

cmd/lotus-worker/sealworker/rpc.go

@@ -26,7 +26,11 @@ import (
 
 var log = logging.Logger("sealworker")
 
-func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permission, error), remote http.HandlerFunc, a api.Worker, permissioned bool) http.Handler {
+func WorkerHandler(
+	authv func(ctx context.Context, token string) ([]auth.Permission, error),
+	remote http.HandlerFunc,
+	a api.Worker,
+	permissioned bool) http.Handler {
 	mux := mux.NewRouter()
 	readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
 	rpcServer := jsonrpc.NewServer(jsonrpc.WithServerErrors(api.RPCErrors), readerServerOpt)

cmd/lotus/daemon.go

@@ -269,6 +269,26 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
+		if cctx.Bool("remove-existing-chain") {
+			lr, err := repo.NewFS(cctx.String("repo"))
+			if err != nil {
+				return xerrors.Errorf("error opening fs repo: %w", err)
+			}
+
+			exists, err := lr.Exists()
+			if err != nil {
+				return err
+			}
+			if !exists {
+				return xerrors.Errorf("lotus repo doesn't exist")
+			}
+
+			err = removeExistingChain(cctx, lr)
+			if err != nil {
+				return err
+			}
+		}
+
 		chainfile := cctx.String("import-chain")
 		snapshot := cctx.String("import-snapshot")
 		willImportChain := false

documentation/en/api-v0-methods-provider.md

@@ -0,0 +1,25 @@
+# Groups
+* [](#)
+  * [Shutdown](#Shutdown)
+  * [Version](#Version)
+## 
+
+
+### Shutdown
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `{}`
+
+### Version
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `131840`
+

documentation/en/api-v1-unstable-methods.md

@@ -211,9 +211,6 @@
   * [PaychVoucherCreate](#PaychVoucherCreate)
   * [PaychVoucherList](#PaychVoucherList)
   * [PaychVoucherSubmit](#PaychVoucherSubmit)
-* [Raft](#Raft)
-  * [RaftLeader](#RaftLeader)
-  * [RaftState](#RaftState)
 * [Start](#Start)
   * [StartTime](#StartTime)
 * [State](#State)
@@ -6182,33 +6179,6 @@ Response:
 }
 ```
 
-## Raft
-
-
-### RaftLeader
-
-
-Perms: read
-
-Inputs: `null`
-
-Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"`
-
-### RaftState
-
-
-Perms: read
-
-Inputs: `null`
-
-Response:
-```json
-{
-  "NonceMap": {},
-  "MsgUuids": {}
-}
-```
-
 ## Start
 
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.3-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -66,6 +66,7 @@ OPTIONS:
    --no-local-storage                                         don't use storageminer repo for sector storage (default: false)
    --gas-premium value                                        set gas premium for initialization messages in AttoFIL (default: "0")
    --from value                                               select which address to send actor creation message from
+   --confidence value                                         number of block confirmations to wait for (default: 5)
    --help, -h                                                 show help
 ```
 
@@ -231,8 +232,19 @@ OPTIONS:
    --help, -h  show help
 ```
 
-#### lotus-miner actor set-addresses, set-addrs
+### lotus-miner actor set-addresses
 ```
+NAME:
+   lotus-miner actor set-addresses - set addresses that your miner can be publicly dialed on
+
+USAGE:
+   lotus-miner actor set-addresses [command options] <multiaddrs>
+
+OPTIONS:
+   --from value       optionally specify the account to send the message from
+   --gas-limit value  set gas limit (default: 0)
+   --unset            unset address (default: false)
+   --help, -h         show help
 ```
 
 ### lotus-miner actor withdraw
@@ -1161,8 +1173,20 @@ OPTIONS:
    --help, -h  show help
 ```
 
-##### lotus-miner proving compute windowed-post, window-post
+#### lotus-miner proving compute windowed-post
 ```
+NAME:
+   lotus-miner proving compute windowed-post - Compute WindowPoSt for a specific deadline
+
+USAGE:
+   lotus-miner proving compute windowed-post [command options] [deadline index]
+
+DESCRIPTION:
+   Note: This command is intended to be used to verify PoSt compute performance.
+   It will not send any messages to the chain.
+
+OPTIONS:
+   --help, -h  show help
 ```
 
 ### lotus-miner proving recover-faults

documentation/en/cli-lotus-provider.md

@@ -0,0 +1,430 @@
+# lotus-provider
+```
+NAME:
+   lotus-provider - Filecoin decentralized storage network provider
+
+USAGE:
+   lotus-provider [global options] command [command options] [arguments...]
+
+VERSION:
+   1.25.3-dev
+
+COMMANDS:
+   run      Start a lotus provider process
+   stop     Stop a running lotus provider
+   config   Manage node config by layers. The layer 'base' will always be applied. 
+   test     Utility functions for testing
+   web      Start lotus provider web interface
+   version  Print version
+   help, h  Shows a list of commands or help for one command
+   DEVELOPER:
+     auth          Manage RPC permissions
+     log           Manage logging
+     wait-api      Wait for lotus api to come online
+     fetch-params  Fetch proving parameters
+
+GLOBAL OPTIONS:
+   --color              use color in display output (default: depends on output being a TTY)
+   --db-host value      Command separated list of hostnames for yugabyte cluster (default: "yugabyte") [$LOTUS_DB_HOST]
+   --db-name value      (default: "yugabyte") [$LOTUS_DB_NAME, $LOTUS_HARMONYDB_HOSTS]
+   --db-user value      (default: "yugabyte") [$LOTUS_DB_USER, $LOTUS_HARMONYDB_USERNAME]
+   --db-password value  (default: "yugabyte") [$LOTUS_DB_PASSWORD, $LOTUS_HARMONYDB_PASSWORD]
+   --layers value       (default: "base") [$LOTUS_LAYERS, $LOTUS_CONFIG_LAYERS]
+   --repo-path value    (default: "~/.lotusprovider") [$LOTUS_REPO_PATH]
+   --vv                 enables very verbose mode, useful for debugging the CLI (default: false)
+   --help, -h           show help
+   --version, -v        print the version
+```
+
+## lotus-provider run
+```
+NAME:
+   lotus-provider run - Start a lotus provider process
+
+USAGE:
+   lotus-provider run [command options] [arguments...]
+
+OPTIONS:
+   --listen value                     host address and port the worker api will listen on (default: "0.0.0.0:12300") [$LOTUS_WORKER_LISTEN]
+   --nosync                           don't check full-node sync status (default: false)
+   --manage-fdlimit                   manage open file limit (default: true)
+   --layers value [ --layers value ]  list of layers to be interpreted (atop defaults). Default: base (default: "base")
+   --storage-json value               path to json file containing storage config (default: "~/.lotus-provider/storage.json")
+   --journal value                    path to journal files (default: "~/.lotus-provider/")
+   --help, -h                         show help
+```
+
+## lotus-provider stop
+```
+NAME:
+   lotus-provider stop - Stop a running lotus provider
+
+USAGE:
+   lotus-provider stop [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help
+```
+
+## lotus-provider config
+```
+NAME:
+   lotus-provider config - Manage node config by layers. The layer 'base' will always be applied. 
+
+USAGE:
+   lotus-provider config command [command options] [arguments...]
+
+COMMANDS:
+   default, defaults                Print default node config
+   set, add, update, create         Set a config layer or the base by providing a filename or stdin.
+   get, cat, show                   Get a config layer by name. You may want to pipe the output to a file, or use 'less'
+   list, ls                         List config layers you can get.
+   interpret, view, stacked, stack  Interpret stacked config layers by this version of lotus-provider, with system-generated comments.
+   remove, rm, del, delete          Remove a named config layer.
+   from-miner                       Express a database config (for lotus-provider) from an existing miner.
+   help, h                          Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider config default
+```
+NAME:
+   lotus-provider config default - Print default node config
+
+USAGE:
+   lotus-provider config default [command options] [arguments...]
+
+OPTIONS:
+   --no-comment  don't comment default values (default: false)
+   --help, -h    show help
+```
+
+### lotus-provider config set
+```
+NAME:
+   lotus-provider config set - Set a config layer or the base by providing a filename or stdin.
+
+USAGE:
+   lotus-provider config set [command options] a layer's file name
+
+OPTIONS:
+   --title value  title of the config layer (req'd for stdin)
+   --help, -h     show help
+```
+
+### lotus-provider config get
+```
+NAME:
+   lotus-provider config get - Get a config layer by name. You may want to pipe the output to a file, or use 'less'
+
+USAGE:
+   lotus-provider config get [command options] layer name
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider config list
+```
+NAME:
+   lotus-provider config list - List config layers you can get.
+
+USAGE:
+   lotus-provider config list [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider config interpret
+```
+NAME:
+   lotus-provider config interpret - Interpret stacked config layers by this version of lotus-provider, with system-generated comments.
+
+USAGE:
+   lotus-provider config interpret [command options] a list of layers to be interpreted as the final config
+
+OPTIONS:
+   --layers value [ --layers value ]  comma or space separated list of layers to be interpreted (default: "base")
+   --help, -h                         show help
+```
+
+### lotus-provider config remove
+```
+NAME:
+   lotus-provider config remove - Remove a named config layer.
+
+USAGE:
+   lotus-provider config remove [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider config from-miner
+```
+NAME:
+   lotus-provider config from-miner - Express a database config (for lotus-provider) from an existing miner.
+
+USAGE:
+   lotus-provider config from-miner [command options] [arguments...]
+
+DESCRIPTION:
+   Express a database config (for lotus-provider) from an existing miner.
+
+OPTIONS:
+   --miner-repo value, --storagerepo value  Specify miner repo path. flag(storagerepo) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON (default: "~/.lotusminer") [$LOTUS_MINER_PATH, $LOTUS_STORAGE_PATH]
+   --to-layer value, -t value               The layer name for this data push. 'base' is recommended for single-miner setup.
+   --overwrite, -o                          Use this with --to-layer to replace an existing layer (default: false)
+   --help, -h                               show help
+```
+
+## lotus-provider test
+```
+NAME:
+   lotus-provider test - Utility functions for testing
+
+USAGE:
+   lotus-provider test command [command options] [arguments...]
+
+COMMANDS:
+   window-post, wd, windowpost, wdpost  Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain.
+   help, h                              Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider test window-post
+```
+NAME:
+   lotus-provider test window-post - Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed). These will not send to the chain.
+
+USAGE:
+   lotus-provider test window-post command [command options] [arguments...]
+
+COMMANDS:
+   here, cli                                       Compute WindowPoSt for performance and configuration testing.
+   task, scheduled, schedule, async, asynchronous  Test the windowpost scheduler by running it on the next available lotus-provider. 
+   help, h                                         Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help
+```
+
+#### lotus-provider test window-post here
+```
+NAME:
+   lotus-provider test window-post here - Compute WindowPoSt for performance and configuration testing.
+
+USAGE:
+   lotus-provider test window-post here [command options] [deadline index]
+
+DESCRIPTION:
+   Note: This command is intended to be used to verify PoSt compute performance.
+   It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.
+
+OPTIONS:
+   --deadline value                   deadline to compute WindowPoSt for  (default: 0)
+   --layers value [ --layers value ]  list of layers to be interpreted (atop defaults). Default: base (default: "base")
+   --storage-json value               path to json file containing storage config (default: "~/.lotus-provider/storage.json")
+   --partition value                  partition to compute WindowPoSt for (default: 0)
+   --help, -h                         show help
+```
+
+#### lotus-provider test window-post task
+```
+NAME:
+   lotus-provider test window-post task - Test the windowpost scheduler by running it on the next available lotus-provider. 
+
+USAGE:
+   lotus-provider test window-post task [command options] [arguments...]
+
+OPTIONS:
+   --deadline value                   deadline to compute WindowPoSt for  (default: 0)
+   --layers value [ --layers value ]  list of layers to be interpreted (atop defaults). Default: base (default: "base")
+   --help, -h                         show help
+```
+
+## lotus-provider web
+```
+NAME:
+   lotus-provider web - Start lotus provider web interface
+
+USAGE:
+   lotus-provider web [command options] [arguments...]
+
+DESCRIPTION:
+   Start an instance of lotus provider web interface. 
+     This creates the 'web' layer if it does not exist, then calls run with that layer.
+
+OPTIONS:
+   --listen value                     Address to listen on (default: "127.0.0.1:4701")
+   --layers value [ --layers value ]  list of layers to be interpreted (atop defaults). Default: base. Web will be added (default: "base")
+   --nosync                           don't check full-node sync status (default: false)
+   --help, -h                         show help
+```
+
+## lotus-provider version
+```
+NAME:
+   lotus-provider version - Print version
+
+USAGE:
+   lotus-provider version [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help
+```
+
+## lotus-provider auth
+```
+NAME:
+   lotus-provider auth - Manage RPC permissions
+
+USAGE:
+   lotus-provider auth command [command options] [arguments...]
+
+COMMANDS:
+   create-token  Create token
+   api-info      Get token with API info required to connect to this node
+   help, h       Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider auth create-token
+```
+NAME:
+   lotus-provider auth create-token - Create token
+
+USAGE:
+   lotus-provider auth create-token [command options] [arguments...]
+
+OPTIONS:
+   --perm value  permission to assign to the token, one of: read, write, sign, admin
+   --help, -h    show help
+```
+
+### lotus-provider auth api-info
+```
+NAME:
+   lotus-provider auth api-info - Get token with API info required to connect to this node
+
+USAGE:
+   lotus-provider auth api-info [command options] [arguments...]
+
+OPTIONS:
+   --perm value  permission to assign to the token, one of: read, write, sign, admin
+   --help, -h    show help
+```
+
+## lotus-provider log
+```
+NAME:
+   lotus-provider log - Manage logging
+
+USAGE:
+   lotus-provider log command [command options] [arguments...]
+
+COMMANDS:
+   list       List log systems
+   set-level  Set log level
+   alerts     Get alert states
+   help, h    Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider log list
+```
+NAME:
+   lotus-provider log list - List log systems
+
+USAGE:
+   lotus-provider log list [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help
+```
+
+### lotus-provider log set-level
+```
+NAME:
+   lotus-provider log set-level - Set log level
+
+USAGE:
+   lotus-provider log set-level [command options] [level]
+
+DESCRIPTION:
+   Set the log level for logging systems:
+
+      The system flag can be specified multiple times.
+
+      eg) log set-level --system chain --system chainxchg debug
+
+      Available Levels:
+      debug
+      info
+      warn
+      error
+
+      Environment Variables:
+      GOLOG_LOG_LEVEL - Default log level for all log systems
+      GOLOG_LOG_FMT   - Change output log format (json, nocolor)
+      GOLOG_FILE      - Write logs to file
+      GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
+
+
+OPTIONS:
+   --system value [ --system value ]  limit to log system
+   --help, -h                         show help
+```
+
+### lotus-provider log alerts
+```
+NAME:
+   lotus-provider log alerts - Get alert states
+
+USAGE:
+   lotus-provider log alerts [command options] [arguments...]
+
+OPTIONS:
+   --all       get all (active and inactive) alerts (default: false)
+   --help, -h  show help
+```
+
+## lotus-provider wait-api
+```
+NAME:
+   lotus-provider wait-api - Wait for lotus api to come online
+
+USAGE:
+   lotus-provider wait-api [command options] [arguments...]
+
+CATEGORY:
+   DEVELOPER
+
+OPTIONS:
+   --timeout value  duration to wait till fail (default: 30s)
+   --help, -h       show help
+```
+
+## lotus-provider fetch-params
+```
+NAME:
+   lotus-provider fetch-params - Fetch proving parameters
+
+USAGE:
+   lotus-provider fetch-params [command options] [sectorSize]
+
+CATEGORY:
+   DEVELOPER
+
+OPTIONS:
+   --help, -h  show help
+```

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.3-dev
 
 COMMANDS:
    run        Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.3-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process
@@ -1807,8 +1807,16 @@ OPTIONS:
    --help, -h   show help
 ```
 
-#### lotus state sector, sector-info
+### lotus state sector
 ```
+NAME:
+   lotus state sector - Get miner sector info
+
+USAGE:
+   lotus state sector [command options] [minerAddress] [sectorNumber]
+
+OPTIONS:
+   --help, -h  show help
 ```
 
 ### lotus state get-actor
@@ -1937,12 +1945,29 @@ OPTIONS:
    --help, -h  show help
 ```
 
-#### lotus state wait-msg, wait-message
+### lotus state wait-msg
 ```
+NAME:
+   lotus state wait-msg - Wait for a message to appear on chain
+
+USAGE:
+   lotus state wait-msg [command options] [messageCid]
+
+OPTIONS:
+   --timeout value  (default: "10m")
+   --help, -h       show help
 ```
 
-#### lotus state search-msg, search-message
+### lotus state search-msg
 ```
+NAME:
+   lotus state search-msg - Search to see whether a message has appeared on chain
+
+USAGE:
+   lotus state search-msg [command options] [messageCid]
+
+OPTIONS:
+   --help, -h  show help
 ```
 
 ### lotus state miner-info
@@ -2080,8 +2105,17 @@ OPTIONS:
    --help, -h  show help
 ```
 
-#### lotus chain get-block, getblock
+### lotus chain get-block
 ```
+NAME:
+   lotus chain get-block - Get a block and print its details
+
+USAGE:
+   lotus chain get-block [command options] [blockCid]
+
+OPTIONS:
+   --raw       print just the raw block header (default: false)
+   --help, -h  show help
 ```
 
 ### lotus chain read-obj
@@ -2132,16 +2166,46 @@ OPTIONS:
    --help, -h    show help
 ```
 
-##### lotus chain getmessage, get-message, get-msg
+### lotus chain getmessage
 ```
+NAME:
+   lotus chain getmessage - Get and print a message by its cid
+
+USAGE:
+   lotus chain getmessage [command options] [messageCid]
+
+OPTIONS:
+   --help, -h  show help
 ```
 
-#### lotus chain sethead, set-head
+### lotus chain sethead
 ```
+NAME:
+   lotus chain sethead - manually set the local nodes head tipset (Caution: normally only used for recovery)
+
+USAGE:
+   lotus chain sethead [command options] [tipsetkey]
+
+OPTIONS:
+   --genesis      reset head to genesis (default: false)
+   --epoch value  reset head to given epoch (default: 0)
+   --help, -h     show help
 ```
 
-#### lotus chain list, love
+### lotus chain list
 ```
+NAME:
+   lotus chain list - View a segment of the chain
+
+USAGE:
+   lotus chain list [command options] [arguments...]
+
+OPTIONS:
+   --height value  (default: current head)
+   --count value   (default: 30)
+   --format value  specify the format to print out tipsets (default: "<height>: (<time>) <blocks>")
+   --gas-stats     view gas statistics for the chain (default: false)
+   --help, -h      show help
 ```
 
 ### lotus chain get
@@ -2768,8 +2832,16 @@ OPTIONS:
    --help, -h  show help
 ```
 
-#### lotus net find-peer, findpeer
+### lotus net find-peer
 ```
+NAME:
+   lotus net find-peer - Find the addresses of a given peerID
+
+USAGE:
+   lotus net find-peer [command options] [peerId]
+
+OPTIONS:
+   --help, -h  show help
 ```
 
 ### lotus net scores

documentation/en/default-lotus-config.toml

@@ -260,68 +260,6 @@
     #HotstoreMaxSpaceSafetyBuffer = 50000000000
 
 
-[Cluster]
-  # EXPERIMENTAL. config to enabled node cluster with raft consensus
-  #
-  # type: bool
-  # env var: LOTUS_CLUSTER_CLUSTERMODEENABLED
-  #ClusterModeEnabled = false
-
-  # A folder to store Raft's data.
-  #
-  # type: string
-  # env var: LOTUS_CLUSTER_DATAFOLDER
-  #DataFolder = ""
-
-  # InitPeersetMultiAddr provides the list of initial cluster peers for new Raft
-  # peers (with no prior state). It is ignored when Raft was already
-  # initialized or when starting in staging mode.
-  #
-  # type: []string
-  # env var: LOTUS_CLUSTER_INITPEERSETMULTIADDR
-  #InitPeersetMultiAddr = []
-
-  # LeaderTimeout specifies how long to wait for a leader before
-  # failing an operation.
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_WAITFORLEADERTIMEOUT
-  #WaitForLeaderTimeout = "15s"
-
-  # NetworkTimeout specifies how long before a Raft network
-  # operation is timed out
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_NETWORKTIMEOUT
-  #NetworkTimeout = "1m40s"
-
-  # CommitRetries specifies how many times we retry a failed commit until
-  # we give up.
-  #
-  # type: int
-  # env var: LOTUS_CLUSTER_COMMITRETRIES
-  #CommitRetries = 1
-
-  # How long to wait between retries
-  #
-  # type: Duration
-  # env var: LOTUS_CLUSTER_COMMITRETRYDELAY
-  #CommitRetryDelay = "200ms"
-
-  # BackupsRotate specifies the maximum number of Raft's DataFolder
-  # copies that we keep as backups (renaming) after cleanup.
-  #
-  # type: int
-  # env var: LOTUS_CLUSTER_BACKUPSROTATE
-  #BackupsRotate = 6
-
-  # Tracing enables propagation of contexts across binary boundaries.
-  #
-  # type: bool
-  # env var: LOTUS_CLUSTER_TRACING
-  #Tracing = false
-
-
 [Fevm]
   # EnableEthRPC enables eth_ rpc, and enables storing a mapping of eth transaction hashes to filecoin message Cids.
   # This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default, but they can be disabled by config options above.

documentation/en/default-lotus-miner-config.toml

@@ -145,6 +145,14 @@
   # env var: LOTUS_SUBSYSTEMS_ENABLEMARKETS
   #EnableMarkets = false
 
+  # When enabled, the sector index will reside in an external database
+  # as opposed to the local KV store in the miner process
+  # This is useful to allow workers to bypass the lotus miner to access sector information
+  #
+  # type: bool
+  # env var: LOTUS_SUBSYSTEMS_ENABLESECTORINDEXDB
+  #EnableSectorIndexDB = false
+
   # type: string
   # env var: LOTUS_SUBSYSTEMS_SEALERAPIINFO
   #SealerApiInfo = ""
@@ -153,6 +161,31 @@
   # env var: LOTUS_SUBSYSTEMS_SECTORINDEXAPIINFO
   #SectorIndexApiInfo = ""
 
+  # When window post is enabled, the miner will automatically submit window post proofs
+  # for all sectors that are eligible for window post
+  # IF WINDOW POST IS DISABLED, THE MINER WILL NOT SUBMIT WINDOW POST PROOFS
+  # THIS WILL RESULT IN FAULTS AND PENALTIES IF NO OTHER MECHANISM IS RUNNING
+  # TO SUBMIT WINDOW POST PROOFS.
+  # Note: This option is entirely disabling the window post scheduler,
+  # not just the builtin PoSt computation like Proving.DisableBuiltinWindowPoSt.
+  # This option will stop lotus-miner from performing any actions related
+  # to window post, including scheduling, submitting proofs, and recovering
+  # sectors.
+  #
+  # type: bool
+  # env var: LOTUS_SUBSYSTEMS_DISABLEWINDOWPOST
+  #DisableWindowPoSt = false
+
+  # When winning post is disabled, the miner process will NOT attempt to mine
+  # blocks. This should only be set when there's an external process mining
+  # blocks on behalf of the miner.
+  # When disabled and no external block producers are configured, all potential
+  # block rewards will be missed!
+  #
+  # type: bool
+  # env var: LOTUS_SUBSYSTEMS_DISABLEWINNINGPOST
+  #DisableWinningPoSt = false
+
 
 [Dealmaking]
   # When enabled, the miner can accept online deals
@@ -896,3 +929,36 @@
   #GCInterval = "1m0s"
 
 
+[HarmonyDB]
+  # HOSTS is a list of hostnames to nodes running YugabyteDB
+  # in a cluster. Only 1 is required
+  #
+  # type: []string
+  # env var: LOTUS_HARMONYDB_HOSTS
+  #Hosts = ["127.0.0.1"]
+
+  # The Yugabyte server's username with full credentials to operate on Lotus' Database. Blank for default.
+  #
+  # type: string
+  # env var: LOTUS_HARMONYDB_USERNAME
+  #Username = "yugabyte"
+
+  # The password for the related username. Blank for default.
+  #
+  # type: string
+  # env var: LOTUS_HARMONYDB_PASSWORD
+  #Password = "yugabyte"
+
+  # The database (logical partition) within Yugabyte. Blank for default.
+  #
+  # type: string
+  # env var: LOTUS_HARMONYDB_DATABASE
+  #Database = "yugabyte"
+
+  # The port to find Yugabyte. Blank for default.
+  #
+  # type: string
+  # env var: LOTUS_HARMONYDB_PORT
+  #Port = "5433"
+
+

documentation/en/default-lotus-provider-config.toml

@@ -0,0 +1,217 @@
+[Subsystems]
+  # type: bool
+  #EnableWindowPost = false
+
+  # type: int
+  #WindowPostMaxTasks = 0
+
+  # type: bool
+  #EnableWinningPost = false
+
+  # type: int
+  #WinningPostMaxTasks = 0
+
+  # type: bool
+  #EnableWebGui = false
+
+  # The address that should listen for Web GUI requests.
+  #
+  # type: string
+  #GuiAddress = ":4701"
+
+
+[Fees]
+  # type: types.FIL
+  #DefaultMaxFee = "0.07 FIL"
+
+  # type: types.FIL
+  #MaxPreCommitGasFee = "0.025 FIL"
+
+  # type: types.FIL
+  #MaxCommitGasFee = "0.05 FIL"
+
+  # type: types.FIL
+  #MaxTerminateGasFee = "0.5 FIL"
+
+  # WindowPoSt is a high-value operation, so the default fee should be high.
+  #
+  # type: types.FIL
+  #MaxWindowPoStGasFee = "5 FIL"
+
+  # type: types.FIL
+  #MaxPublishDealsFee = "0.05 FIL"
+
+  [Fees.MaxPreCommitBatchGasFee]
+    # type: types.FIL
+    #Base = "0 FIL"
+
+    # type: types.FIL
+    #PerSector = "0.02 FIL"
+
+  [Fees.MaxCommitBatchGasFee]
+    # type: types.FIL
+    #Base = "0 FIL"
+
+    # type: types.FIL
+    #PerSector = "0.03 FIL"
+
+
+[Addresses]
+  # Addresses to send PreCommit messages from
+  #
+  # type: []string
+  #PreCommitControl = []
+
+  # Addresses to send Commit messages from
+  #
+  # type: []string
+  #CommitControl = []
+
+  # type: []string
+  #TerminateControl = []
+
+  # DisableOwnerFallback disables usage of the owner address for messages
+  # sent automatically
+  #
+  # type: bool
+  #DisableOwnerFallback = false
+
+  # DisableWorkerFallback disables usage of the worker address for messages
+  # sent automatically, if control addresses are configured.
+  # A control address that doesn't have enough funds will still be chosen
+  # over the worker address if this flag is set.
+  #
+  # type: bool
+  #DisableWorkerFallback = false
+
+
+[Proving]
+  # Maximum number of sector checks to run in parallel. (0 = unlimited)
+  # 
+  # WARNING: Setting this value too high may make the node crash by running out of stack
+  # WARNING: Setting this value too low may make sector challenge reading much slower, resulting in failed PoSt due
+  # to late submission.
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: int
+  #ParallelCheckLimit = 32
+
+  # Maximum amount of time a proving pre-check can take for a sector. If the check times out the sector will be skipped
+  # 
+  # WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+  # test challenge took longer than this timeout
+  # WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this sector are
+  # blocked (e.g. in case of disconnected NFS mount)
+  #
+  # type: Duration
+  #SingleCheckTimeout = "10m0s"
+
+  # Maximum amount of time a proving pre-check can take for an entire partition. If the check times out, sectors in
+  # the partition which didn't get checked on time will be skipped
+  # 
+  # WARNING: Setting this value too low risks in sectors being skipped even though they are accessible, just reading the
+  # test challenge took longer than this timeout
+  # WARNING: Setting this value too high risks missing PoSt deadline in case IO operations related to this partition are
+  # blocked or slow
+  #
+  # type: Duration
+  #PartitionCheckTimeout = "20m0s"
+
+  # Disable Window PoSt computation on the lotus-miner process even if no window PoSt workers are present.
+  # 
+  # WARNING: If no windowPoSt workers are connected, window PoSt WILL FAIL resulting in faulty sectors which will need
+  # to be recovered. Before enabling this option, make sure your PoSt workers work correctly.
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: bool
+  #DisableBuiltinWindowPoSt = false
+
+  # Disable Winning PoSt computation on the lotus-miner process even if no winning PoSt workers are present.
+  # 
+  # WARNING: If no WinningPoSt workers are connected, Winning PoSt WILL FAIL resulting in lost block rewards.
+  # Before enabling this option, make sure your PoSt workers work correctly.
+  #
+  # type: bool
+  #DisableBuiltinWinningPoSt = false
+
+  # Disable WindowPoSt provable sector readability checks.
+  # 
+  # In normal operation, when preparing to compute WindowPoSt, lotus-miner will perform a round of reading challenges
+  # from all sectors to confirm that those sectors can be proven. Challenges read in this process are discarded, as
+  # we're only interested in checking that sector data can be read.
+  # 
+  # When using builtin proof computation (no PoSt workers, and DisableBuiltinWindowPoSt is set to false), this process
+  # can save a lot of time and compute resources in the case that some sectors are not readable - this is caused by
+  # the builtin logic not skipping snark computation when some sectors need to be skipped.
+  # 
+  # When using PoSt workers, this process is mostly redundant, with PoSt workers challenges will be read once, and
+  # if challenges for some sectors aren't readable, those sectors will just get skipped.
+  # 
+  # Disabling sector pre-checks will slightly reduce IO load when proving sectors, possibly resulting in shorter
+  # time to produce window PoSt. In setups with good IO capabilities the effect of this option on proving time should
+  # be negligible.
+  # 
+  # NOTE: It likely is a bad idea to disable sector pre-checks in setups with no PoSt workers.
+  # 
+  # NOTE: Even when this option is enabled, recovering sectors will be checked before recovery declaration message is
+  # sent to the chain
+  # 
+  # After changing this option, confirm that the new value works in your setup by invoking
+  # 'lotus-miner proving compute window-post 0'
+  #
+  # type: bool
+  #DisableWDPoStPreChecks = false
+
+  # Maximum number of partitions to prove in a single SubmitWindowPoSt messace. 0 = network limit (3 in nv21)
+  # 
+  # A single partition may contain up to 2349 32GiB sectors, or 2300 64GiB sectors.
+  # //
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+  # to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
+  # 
+  # Setting this value above the network limit has no effect
+  #
+  # type: int
+  #MaxPartitionsPerPoStMessage = 0
+
+  # In some cases when submitting DeclareFaultsRecovered messages,
+  # there may be too many recoveries to fit in a BlockGasLimit.
+  # In those cases it may be necessary to set this value to something low (eg 1);
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent than needed,
+  # resulting in more total gas use (but each message will have lower gas limit)
+  #
+  # type: int
+  #MaxPartitionsPerRecoveryMessage = 0
+
+  # Enable single partition per PoSt Message for partitions containing recovery sectors
+  # 
+  # In cases when submitting PoSt messages which contain recovering sectors, the default network limit may still be
+  # too high to fit in the block gas limit. In those cases, it becomes useful to only house the single partition
+  # with recovering sectors in the post message
+  # 
+  # Note that setting this value lower may result in less efficient gas use - more messages will be sent,
+  # to prove each deadline, resulting in more total gas use (but each message will have lower gas limit)
+  #
+  # type: bool
+  #SingleRecoveringPartitionPerPostMessage = false
+
+
+[Journal]
+  # Events of the form: "system1:event1,system1:event2[,...]"
+  #
+  # type: string
+  #DisabledEvents = ""
+
+
+[Apis]
+  # RPC Secret for the storage subsystem.
+  # If integrating with lotus-miner this must match the value from
+  # cat ~/.lotusminer/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU | jq -r .PrivateKey
+  #
+  # type: string
+  #StorageRPCSecret = ""
+

gen/main.go

@@ -92,7 +92,7 @@ func main() {
 	err = gen.WriteTupleEncodersToFile("./chain/exchange/cbor_gen.go", "exchange",
 		exchange.Request{},
 		exchange.Response{},
-		exchange.CompactedMessages{},
+		exchange.CompactedMessagesCBOR{},
 		exchange.BSTipSet{},
 	)
 	if err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/filecoin-project/lotus
 
-go 1.19
+go 1.20
 
 retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
 
@@ -62,6 +62,7 @@ require (
 	github.com/filecoin-project/test-vectors/schema v0.0.7
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
 	github.com/gdamore/tcell/v2 v2.2.0
+	github.com/georgysavva/scany/v2 v2.0.0
 	github.com/go-openapi/spec v0.19.11
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
@@ -70,12 +71,9 @@ require (
 	github.com/gregdhill/go-openrpc v0.0.0-20220114144539-ae6f44720487
 	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
-	github.com/hashicorp/go-hclog v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.5
 	github.com/hashicorp/golang-lru/v2 v2.0.5
-	github.com/hashicorp/raft v1.3.10
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
 	github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94
 	github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab
 	github.com/ipfs/bbloom v0.0.4
@@ -104,15 +102,14 @@ require (
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
 	github.com/ipni/go-libipni v0.0.8
 	github.com/ipni/index-provider v0.12.0
+	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
+	github.com/jackc/pgx/v5 v5.4.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
-	github.com/libp2p/go-libp2p v0.31.0
-	github.com/libp2p/go-libp2p-consensus v0.0.1
-	github.com/libp2p/go-libp2p-gorpc v0.5.0
+	github.com/libp2p/go-libp2p v0.31.1
 	github.com/libp2p/go-libp2p-kad-dht v0.24.0
 	github.com/libp2p/go-libp2p-pubsub v0.9.3
-	github.com/libp2p/go-libp2p-raft v0.4.0
 	github.com/libp2p/go-libp2p-record v0.2.0
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.0
 	github.com/libp2p/go-maddr-filter v0.1.0
@@ -122,21 +119,24 @@ require (
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-base32 v0.1.0
-	github.com/multiformats/go-multiaddr v0.11.0
+	github.com/multiformats/go-multiaddr v0.12.1
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multibase v0.2.0
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/multiformats/go-varint v0.0.7
 	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
+	github.com/pkg/errors v0.9.1
 	github.com/polydawn/refmt v0.89.0
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.16.0
 	github.com/puzpuzpuz/xsync/v2 v2.4.0
 	github.com/raulk/clock v1.1.0
 	github.com/raulk/go-watchdog v1.3.0
+	github.com/samber/lo v1.38.1
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7
+	github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed
 	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
@@ -154,12 +154,11 @@ require (
 	go.uber.org/fx v1.20.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.25.0
-	golang.org/x/crypto v0.12.0
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
+	golang.org/x/crypto v0.17.0
 	golang.org/x/net v0.14.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/sys v0.11.0
-	golang.org/x/term v0.11.0
+	golang.org/x/sys v0.15.0
+	golang.org/x/term v0.15.0
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
 	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
@@ -174,11 +173,9 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akavel/rsrc v0.8.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
-	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
@@ -225,8 +222,6 @@ require (
 	github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/huin/goupnp v1.2.0 // indirect
 	github.com/iancoleman/orderedmap v0.1.0 // indirect
@@ -247,6 +242,9 @@ require (
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
 	github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
@@ -288,17 +286,15 @@ require (
 	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.38.1 // indirect
+	github.com/quic-go/quic-go v0.38.2 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
@@ -308,7 +304,6 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
-	github.com/ugorji/go/codec v1.2.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.0.1 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
@@ -324,8 +319,9 @@ require (
 	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
 	google.golang.org/grpc v1.55.0 // indirect
@@ -339,5 +335,3 @@ require (
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
-
-replace github.com/triplewz/poseidon => github.com/magik6k/poseidon v0.0.0-neptune

go.sum

@@ -48,8 +48,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -99,9 +97,6 @@ github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
-github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18=
-github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
@@ -118,8 +113,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
 github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@@ -164,11 +157,10 @@ github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4=
 github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
@@ -393,6 +385,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4=
 github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
+github.com/georgysavva/scany/v2 v2.0.0 h1:RGXqxDv4row7/FYoK8MRXAZXqoWF/NM+NP0q50k3DKU=
+github.com/georgysavva/scany/v2 v2.0.0/go.mod h1:sigOdh+0qb/+aOs3TVhehVT10p8qJL7K/Zhyz8vWo38=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
@@ -450,6 +444,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -588,26 +583,16 @@ github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-hclog v1.3.0 h1:G0ACM8Z2WilWgPv3Vdzwm3V0BQu/kSmrkVtpe1fy9do=
-github.com/hashicorp/go-hclog v1.3.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
-github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
-github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@@ -624,10 +609,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/raft v1.3.10 h1:LR5QZX1VQd0DFWZfeCwWawyeKfpS/Tm1yjnJIY5X4Tw=
-github.com/hashicorp/raft v1.3.10/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
@@ -855,6 +836,16 @@ github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwS
 github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
+github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw=
+github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.4.1 h1:oKfB/FhuVtit1bBM3zNRRsZ925ZkMN3HXL+LgLUM9lE=
+github.com/jackc/pgx/v5 v5.4.1/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
+github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk=
+github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -888,7 +879,6 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -945,6 +935,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
 github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
 github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
 github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
@@ -969,8 +960,8 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS
 github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw=
 github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o=
 github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0=
-github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg=
-github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg=
+github.com/libp2p/go-libp2p v0.31.1 h1:mUiFPwdzC2zMLIATKVddjCuPXVbtC3BsKKVPMs4+jzY=
+github.com/libp2p/go-libp2p v0.31.1/go.mod h1:+9TCv+XySSOdaxPF1WIgTK8rXP9jBb8WbemlMCSXGsU=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
@@ -986,8 +977,6 @@ github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFk
 github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
 github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo=
 github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA=
-github.com/libp2p/go-libp2p-consensus v0.0.1 h1:jcVbHRZLwTXU9iT/mPi+Lx4/OrIzq3bU1TbZNhYFCV8=
-github.com/libp2p/go-libp2p-consensus v0.0.1/go.mod h1:+9Wrfhc5QOqWB0gXI0m6ARlkHfdJpcFXmRU0WoHz4Mo=
 github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
 github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
 github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
@@ -1016,8 +1005,6 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT
 github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg=
 github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw=
 github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug=
-github.com/libp2p/go-libp2p-gorpc v0.5.0 h1:mmxxAPdP3JzpYH4KcDf4csXnqtd1HazLPfdyB2MBRb8=
-github.com/libp2p/go-libp2p-gorpc v0.5.0/go.mod h1:GpHuvY3m0YFkd0+inOGo4HDtc4up9OS/mBPXvEpNuRY=
 github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=
 github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA=
 github.com/libp2p/go-libp2p-kad-dht v0.24.0 h1:nZnFDQEFU4N8GzclnR+IGxIgR7k4PPCDk/GK9A28onk=
@@ -1048,8 +1035,6 @@ github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYc
 github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo=
 github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc=
 github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA=
-github.com/libp2p/go-libp2p-raft v0.4.0 h1:2atEs7/FWH35bRiLh8gTejGh5NA9u4eG7BXjpf/B+Z4=
-github.com/libp2p/go-libp2p-raft v0.4.0/go.mod h1:qJCYtFBTbip2wngLxFeAb9o52XmAPi2vSIQ4hV7IpSA=
 github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
@@ -1163,8 +1148,6 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
 github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magik6k/poseidon v0.0.0-neptune h1:Dfz15iiYGGE9Esvn8pZFlbiiCrHuyZDxm6LGXQfaf9c=
-github.com/magik6k/poseidon v0.0.0-neptune/go.mod h1:QYG1d0B4YZD7TgF6qZndTTu4rxUGFCCZAQRDanDj+9c=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -1183,7 +1166,6 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -1267,8 +1249,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u
 github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI=
 github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc=
 github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0=
-github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10=
-github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM=
+github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk=
+github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE=
 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
@@ -1376,8 +1358,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -1408,15 +1388,14 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
-github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1430,7 +1409,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7q
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@@ -1450,8 +1428,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0=
 github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=
 github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag=
@@ -1460,8 +1438,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
 github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
 github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
-github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
+github.com/quic-go/quic-go v0.38.2 h1:VWv/6gxIoB8hROQJhx1JEyiegsUQ+zMN3em3kynTGdg=
+github.com/quic-go/quic-go v0.38.2/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
 github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=
@@ -1486,6 +1464,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
+github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8=
@@ -1591,15 +1571,13 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed h1:C8H2ql+vCBhEi7d3vMBBbdCAKv9s/thfPyLEuSvFpMU=
+github.com/triplewz/poseidon v0.0.0-20230828015038-79d8165c88ed/go.mod h1:QYG1d0B4YZD7TgF6qZndTTu4rxUGFCCZAQRDanDj+9c=
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
-github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
-github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -1801,8 +1779,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -2022,11 +2000,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -2038,16 +2014,16 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2057,8 +2033,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

itests/harmonydb_test.go

@@ -0,0 +1,174 @@
+package itests
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+func withSetup(t *testing.T, f func(*kit.TestMiner)) {
+	_, miner, _ := kit.EnsembleMinimal(t,
+		kit.LatestActorsAt(-1),
+		kit.MockProofs(),
+		kit.WithSectorIndexDB(),
+	)
+
+	f(miner)
+}
+
+func TestCrud(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	withSetup(t, func(miner *kit.TestMiner) {
+		cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		_, err := cdb.Exec(ctx, `
+			INSERT INTO 
+				itest_scratch (some_int, content) 
+			VALUES 
+				(11, 'cows'), 
+				(5, 'cats')
+		`)
+		if err != nil {
+			t.Fatal("Could not insert: ", err)
+		}
+		var ints []struct {
+			Count       int    `db:"some_int"`
+			Animal      string `db:"content"`
+			Unpopulated int
+		}
+		err = cdb.Select(ctx, &ints, "SELECT content, some_int FROM itest_scratch")
+		if err != nil {
+			t.Fatal("Could not select: ", err)
+		}
+		if len(ints) != 2 {
+			t.Fatal("unexpected count of returns. Want 2, Got ", len(ints))
+		}
+		if ints[0].Count != 11 || ints[1].Count != 5 {
+			t.Fatal("expected [11,5] got ", ints)
+		}
+		if ints[0].Animal != "cows" || ints[1].Animal != "cats" {
+			t.Fatal("expected, [cows, cats] ", ints)
+		}
+		fmt.Println("test completed")
+	})
+}
+
+func TestTransaction(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	withSetup(t, func(miner *kit.TestMiner) {
+		cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		if _, err := cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int) VALUES (4), (5), (6)"); err != nil {
+			t.Fatal("E0", err)
+		}
+		_, err := cdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
+			if _, err := tx.Exec("INSERT INTO itest_scratch (some_int) VALUES (7), (8), (9)"); err != nil {
+				t.Fatal("E1", err)
+			}
+
+			// sum1 is read from OUTSIDE the transaction so it's the old value
+			var sum1 int
+			if err := cdb.QueryRow(ctx, "SELECT SUM(some_int) FROM itest_scratch").Scan(&sum1); err != nil {
+				t.Fatal("E2", err)
+			}
+			if sum1 != 4+5+6 {
+				t.Fatal("Expected 15, got ", sum1)
+			}
+
+			// sum2 is from INSIDE the transaction, so the updated value.
+			var sum2 int
+			if err := tx.QueryRow("SELECT SUM(some_int) FROM itest_scratch").Scan(&sum2); err != nil {
+				t.Fatal("E3", err)
+			}
+			if sum2 != 4+5+6+7+8+9 {
+				t.Fatal("Expected 39, got ", sum2)
+			}
+			return false, nil // rollback
+		})
+		if err != nil {
+			t.Fatal("ET", err)
+		}
+
+		var sum2 int
+		// Query() example (yes, QueryRow would be preferred here)
+		q, err := cdb.Query(ctx, "SELECT SUM(some_int) FROM itest_scratch")
+		if err != nil {
+			t.Fatal("E4", err)
+		}
+		defer q.Close()
+		var rowCt int
+		for q.Next() {
+			err := q.Scan(&sum2)
+			if err != nil {
+				t.Fatal("error scanning ", err)
+			}
+			rowCt++
+		}
+		if sum2 != 4+5+6 {
+			t.Fatal("Expected 15, got ", sum2)
+		}
+		if rowCt != 1 {
+			t.Fatal("unexpected count of rows")
+		}
+	})
+}
+
+func TestPartialWalk(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	withSetup(t, func(miner *kit.TestMiner) {
+		cdb := miner.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		if _, err := cdb.Exec(ctx, `
+			INSERT INTO 
+				itest_scratch (content, some_int) 
+			VALUES 
+				('andy was here', 5), 
+				('lotus is awesome', 6), 
+				('hello world', 7),
+				('3rd integration test', 8),
+				('fiddlesticks', 9)
+			`); err != nil {
+			t.Fatal("e1", err)
+		}
+
+		// TASK: FIND THE ID of the string with a specific SHA256
+		needle := "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
+		q, err := cdb.Query(ctx, `SELECT id, content FROM itest_scratch`)
+		if err != nil {
+			t.Fatal("e2", err)
+		}
+		defer q.Close()
+
+		var tmp struct {
+			Src string `db:"content"`
+			ID  int
+		}
+
+		var done bool
+		for q.Next() {
+
+			if err := q.StructScan(&tmp); err != nil {
+				t.Fatal("structscan err " + err.Error())
+			}
+
+			bSha := sha256.Sum256([]byte(tmp.Src))
+			if hex.EncodeToString(bSha[:]) == needle {
+				done = true
+				break
+			}
+		}
+		if !done {
+			t.Fatal("We didn't find it.")
+		}
+		// Answer: tmp.ID
+	})
+}

itests/harmonytask_test.go

@@ -0,0 +1,266 @@
+package itests
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
+	"github.com/filecoin-project/lotus/lib/harmony/resources"
+	"github.com/filecoin-project/lotus/node/impl"
+)
+
+type task1 struct {
+	toAdd               []int
+	myPersonalTableLock sync.Mutex
+	myPersonalTable     map[harmonytask.TaskID]int // This would typically be a DB table
+	WorkCompleted       []string
+}
+
+func withDbSetup(t *testing.T, f func(*kit.TestMiner)) {
+	_, miner, _ := kit.EnsembleMinimal(t,
+		kit.LatestActorsAt(-1),
+		kit.MockProofs(),
+		kit.WithSectorIndexDB(),
+	)
+	logging.SetLogLevel("harmonytask", "debug")
+
+	f(miner)
+}
+
+func (t *task1) Do(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
+	if !stillOwned() {
+		return false, errors.New("Why not still owned?")
+	}
+	t.myPersonalTableLock.Lock()
+	defer t.myPersonalTableLock.Unlock()
+	t.WorkCompleted = append(t.WorkCompleted, fmt.Sprintf("taskResult%d", t.myPersonalTable[tID]))
+	return true, nil
+}
+func (t *task1) CanAccept(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+	return &list[0], nil
+}
+func (t *task1) TypeDetails() harmonytask.TaskTypeDetails {
+	return harmonytask.TaskTypeDetails{
+		Max:         100,
+		Name:        "ThingOne",
+		MaxFailures: 1,
+		Cost: resources.Resources{
+			Cpu: 1,
+			Ram: 100 << 10, // at 100kb, it's tiny
+		},
+	}
+}
+func (t *task1) Adder(add harmonytask.AddTaskFunc) {
+	for _, vTmp := range t.toAdd {
+		v := vTmp
+		add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
+			t.myPersonalTableLock.Lock()
+			defer t.myPersonalTableLock.Unlock()
+
+			t.myPersonalTable[tID] = v
+			return true, nil
+		})
+	}
+}
+
+func init() {
+	//logging.SetLogLevel("harmonydb", "debug")
+	//logging.SetLogLevel("harmonytask", "debug")
+}
+
+func TestHarmonyTasks(t *testing.T) {
+	//t.Parallel()
+	withDbSetup(t, func(m *kit.TestMiner) {
+		cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		t1 := &task1{
+			toAdd:           []int{56, 73},
+			myPersonalTable: map[harmonytask.TaskID]int{},
+		}
+		harmonytask.POLL_DURATION = time.Millisecond * 100
+		e, err := harmonytask.New(cdb, []harmonytask.TaskInterface{t1}, "test:1")
+		require.NoError(t, err)
+		time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
+		e.GracefullyTerminate(time.Minute)
+		expected := []string{"taskResult56", "taskResult73"}
+		sort.Strings(t1.WorkCompleted)
+		require.Equal(t, expected, t1.WorkCompleted, "unexpected results")
+	})
+}
+
+type passthru struct {
+	dtl       harmonytask.TaskTypeDetails
+	do        func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error)
+	canAccept func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error)
+	adder     func(add harmonytask.AddTaskFunc)
+}
+
+func (t *passthru) Do(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
+	return t.do(tID, stillOwned)
+}
+func (t *passthru) CanAccept(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+	return t.canAccept(list, e)
+}
+func (t *passthru) TypeDetails() harmonytask.TaskTypeDetails {
+	return t.dtl
+}
+func (t *passthru) Adder(add harmonytask.AddTaskFunc) {
+	if t.adder != nil {
+		t.adder(add)
+	}
+}
+
+// Common stuff
+var dtl = harmonytask.TaskTypeDetails{Name: "foo", Max: -1, Cost: resources.Resources{}}
+var lettersMutex sync.Mutex
+
+func fooLetterAdder(t *testing.T, cdb *harmonydb.DB) *passthru {
+	return &passthru{
+		dtl: dtl,
+		canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+			return nil, nil
+		},
+		adder: func(add harmonytask.AddTaskFunc) {
+			for _, vTmp := range []string{"A", "B"} {
+				v := vTmp
+				add(func(tID harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) {
+					_, err := tx.Exec("INSERT INTO itest_scratch (some_int, content) VALUES ($1,$2)", tID, v)
+					require.NoError(t, err)
+					return true, nil
+				})
+			}
+		},
+	}
+}
+func fooLetterSaver(t *testing.T, cdb *harmonydb.DB, dest *[]string) *passthru {
+	return &passthru{
+		dtl: dtl,
+		canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+			return &list[0], nil
+		},
+		do: func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
+			var content string
+			err = cdb.QueryRow(context.Background(),
+				"SELECT content FROM itest_scratch WHERE some_int=$1", tID).Scan(&content)
+			require.NoError(t, err)
+			lettersMutex.Lock()
+			defer lettersMutex.Unlock()
+			*dest = append(*dest, content)
+			return true, nil
+		},
+	}
+}
+
+func TestHarmonyTasksWith2PartiesPolling(t *testing.T) {
+	//t.Parallel()
+	withDbSetup(t, func(m *kit.TestMiner) {
+		cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		senderParty := fooLetterAdder(t, cdb)
+		var dest []string
+		workerParty := fooLetterSaver(t, cdb, &dest)
+		harmonytask.POLL_DURATION = time.Millisecond * 100
+		sender, err := harmonytask.New(cdb, []harmonytask.TaskInterface{senderParty}, "test:1")
+		require.NoError(t, err)
+		worker, err := harmonytask.New(cdb, []harmonytask.TaskInterface{workerParty}, "test:2")
+		require.NoError(t, err)
+		time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
+		sender.GracefullyTerminate(time.Second * 5)
+		worker.GracefullyTerminate(time.Second * 5)
+		sort.Strings(dest)
+		require.Equal(t, []string{"A", "B"}, dest)
+	})
+}
+
+func TestWorkStealing(t *testing.T) {
+	//t.Parallel()
+	withDbSetup(t, func(m *kit.TestMiner) {
+		cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		ctx := context.Background()
+
+		// The dead worker will be played by a few SQL INSERTS.
+		_, err := cdb.Exec(ctx, `INSERT INTO harmony_machines
+		(id, last_contact,host_and_port, cpu, ram, gpu)
+		VALUES (300, DATE '2000-01-01', 'test:1', 4, 400000, 1)`)
+		require.ErrorIs(t, err, nil)
+		_, err = cdb.Exec(ctx, `INSERT INTO harmony_task 
+		(id, name, owner_id, posted_time, added_by) 
+		VALUES (1234, 'foo', 300, DATE '2000-01-01', 300)`)
+		require.ErrorIs(t, err, nil)
+		_, err = cdb.Exec(ctx, "INSERT INTO itest_scratch (some_int, content) VALUES (1234, 'M')")
+		require.ErrorIs(t, err, nil)
+
+		harmonytask.POLL_DURATION = time.Millisecond * 100
+		harmonytask.CLEANUP_FREQUENCY = time.Millisecond * 100
+		var dest []string
+		worker, err := harmonytask.New(cdb, []harmonytask.TaskInterface{fooLetterSaver(t, cdb, &dest)}, "test:2")
+		require.ErrorIs(t, err, nil)
+		time.Sleep(time.Second) // do the work. FLAKYNESS RISK HERE.
+		worker.GracefullyTerminate(time.Second * 5)
+		require.Equal(t, []string{"M"}, dest)
+	})
+}
+
+func TestTaskRetry(t *testing.T) {
+	//t.Parallel()
+	withDbSetup(t, func(m *kit.TestMiner) {
+		cdb := m.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB
+		senderParty := fooLetterAdder(t, cdb)
+		harmonytask.POLL_DURATION = time.Millisecond * 100
+		sender, err := harmonytask.New(cdb, []harmonytask.TaskInterface{senderParty}, "test:1")
+		require.NoError(t, err)
+
+		alreadyFailed := map[string]bool{}
+		var dest []string
+		fails2xPerMsg := &passthru{
+			dtl: dtl,
+			canAccept: func(list []harmonytask.TaskID, e *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+				return &list[0], nil
+			},
+			do: func(tID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
+				var content string
+				err = cdb.QueryRow(context.Background(),
+					"SELECT content FROM itest_scratch WHERE some_int=$1", tID).Scan(&content)
+				require.NoError(t, err)
+				lettersMutex.Lock()
+				defer lettersMutex.Unlock()
+				if !alreadyFailed[content] {
+					alreadyFailed[content] = true
+					return false, errors.New("intentional 'error'")
+				}
+				dest = append(dest, content)
+				return true, nil
+			},
+		}
+		rcv, err := harmonytask.New(cdb, []harmonytask.TaskInterface{fails2xPerMsg}, "test:2")
+		require.NoError(t, err)
+		time.Sleep(time.Second)
+		sender.GracefullyTerminate(time.Hour)
+		rcv.GracefullyTerminate(time.Hour)
+		sort.Strings(dest)
+		require.Equal(t, []string{"A", "B"}, dest)
+		type hist struct {
+			TaskID int
+			Result bool
+			Err    string
+		}
+		var res []hist
+		require.NoError(t, cdb.Select(context.Background(), &res,
+			`SELECT task_id, result, err FROM harmony_task_history
+			 ORDER BY result DESC, task_id`))
+
+		require.Equal(t, []hist{
+			{1, true, ""},
+			{2, true, ""},
+			{1, false, "error: intentional 'error'"},
+			{2, false, "error: intentional 'error'"}}, res)
+	})
+}

itests/kit/ensemble.go

@@ -20,6 +20,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/go-address"
 	cborutil "github.com/filecoin-project/go-cbor-util"
@@ -45,15 +46,20 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet/key"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/rpc"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/tasks"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker"
 	"github.com/filecoin-project/lotus/gateway"
 	"github.com/filecoin-project/lotus/genesis"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/markets/idxprov"
 	"github.com/filecoin-project/lotus/markets/idxprov/idxprov_test"
 	lotusminer "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/impl"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	testing2 "github.com/filecoin-project/lotus/node/modules/testing"
@@ -118,15 +124,17 @@ type Ensemble struct {
 	options      *ensembleOpts
 
 	inactive struct {
-		fullnodes []*TestFullNode
-		miners    []*TestMiner
-		workers   []*TestWorker
+		fullnodes     []*TestFullNode
+		providernodes []*TestProviderNode
+		miners        []*TestMiner
+		workers       []*TestWorker
 	}
 	active struct {
-		fullnodes []*TestFullNode
-		miners    []*TestMiner
-		workers   []*TestWorker
-		bms       map[*TestMiner]*BlockMiner
+		fullnodes     []*TestFullNode
+		providernodes []*TestProviderNode
+		miners        []*TestMiner
+		workers       []*TestWorker
+		bms           map[*TestMiner]*BlockMiner
 	}
 	genesis struct {
 		version  network.Version
@@ -219,6 +227,20 @@ func (n *Ensemble) FullNode(full *TestFullNode, opts ...NodeOpt) *Ensemble {
 	return n
 }
 
+// FullNode enrolls a new Provider node.
+func (n *Ensemble) Provider(lp *TestProviderNode, opts ...NodeOpt) *Ensemble {
+	options := DefaultNodeOpts
+	for _, o := range opts {
+		err := o(&options)
+		require.NoError(n.t, err)
+	}
+
+	*lp = TestProviderNode{t: n.t, options: options, Deps: &deps.Deps{}}
+
+	n.inactive.providernodes = append(n.inactive.providernodes, lp)
+	return n
+}
+
 // Miner enrolls a new miner, using the provided full node for chain
 // interactions.
 func (n *Ensemble) MinerEnroll(minerNode *TestMiner, full *TestFullNode, opts ...NodeOpt) *Ensemble {
@@ -359,6 +381,8 @@ func (n *Ensemble) Start() *Ensemble {
 		n.mn = mocknet.New()
 	}
 
+	sharedITestID := harmonydb.ITestNewID()
+
 	// ---------------------
 	//  FULL NODES
 	// ---------------------
@@ -603,6 +627,7 @@ func (n *Ensemble) Start() *Ensemble {
 		cfg.Subsystems.EnableMining = m.options.subsystems.Has(SMining)
 		cfg.Subsystems.EnableSealing = m.options.subsystems.Has(SSealing)
 		cfg.Subsystems.EnableSectorStorage = m.options.subsystems.Has(SSectorStorage)
+		cfg.Subsystems.EnableSectorIndexDB = m.options.subsystems.Has(SHarmony)
 		cfg.Dealmaking.MaxStagingDealsBytes = m.options.maxStagingDealsBytes
 
 		if m.options.mainMiner != nil {
@@ -688,8 +713,6 @@ func (n *Ensemble) Start() *Ensemble {
 		copy.FullNode = modules.MakeUuidWrapper(copy.FullNode)
 		m.FullNode = &copy
 
-		//m.FullNode.FullNode = modules.MakeUuidWrapper(fn.FullNode)
-
 		opts := []node.Option{
 			node.StorageMiner(&m.StorageMiner, cfg.Subsystems),
 			node.Base(),
@@ -697,8 +720,6 @@ func (n *Ensemble) Start() *Ensemble {
 			node.Test(),
 
 			node.If(m.options.disableLibp2p, node.MockHost(n.mn)),
-			//node.Override(new(v1api.RawFullNodeAPI), func() api.FullNode { return modules.MakeUuidWrapper(m.FullNode) }),
-			//node.Override(new(v1api.RawFullNodeAPI), modules.MakeUuidWrapper),
 			node.Override(new(v1api.RawFullNodeAPI), m.FullNode),
 			node.Override(new(*lotusminer.Miner), lotusminer.NewTestMiner(mineBlock, m.ActorAddr)),
 
@@ -724,6 +745,17 @@ func (n *Ensemble) Start() *Ensemble {
 
 			// upgrades
 			node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule),
+
+			node.Override(new(harmonydb.ITestID), sharedITestID),
+			node.Override(new(config.HarmonyDB), func() config.HarmonyDB {
+				return config.HarmonyDB{
+					Hosts:    []string{envElse("LOTUS_HARMONYDB_HOSTS", "127.0.0.1")},
+					Database: "yugabyte",
+					Username: "yugabyte",
+					Password: "yugabyte",
+					Port:     "5433",
+				}
+			}),
 		}
 
 		if m.options.subsystems.Has(SMarkets) {
@@ -770,6 +802,12 @@ func (n *Ensemble) Start() *Ensemble {
 		require.NoError(n.t, err)
 
 		n.t.Cleanup(func() { _ = stop(context.Background()) })
+		mCopy := m
+		n.t.Cleanup(func() {
+			if mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB != nil {
+				mCopy.BaseAPI.(*impl.StorageMinerAPI).HarmonyDB.ITestDeleteAll()
+			}
+		})
 
 		m.BaseAPI = m.StorageMiner
 
@@ -826,6 +864,8 @@ func (n *Ensemble) Start() *Ensemble {
 
 		auth := http.Header(nil)
 
+		// FUTURE: Use m.MinerNode.(BaseAPI).(impl.StorageMinerAPI).HarmonyDB to setup.
+
 		remote := paths.NewRemote(localStore, m.MinerNode, auth, 20, &paths.DefaultPartialFileHandler{})
 		store := m.options.workerStorageOpt(remote)
 
@@ -855,12 +895,35 @@ func (n *Ensemble) Start() *Ensemble {
 		require.NoError(n.t, err)
 
 		n.active.workers = append(n.active.workers, m)
+
 	}
 
 	// If we are here, we have processed all inactive workers and moved them
 	// to active, so clear the slice.
 	n.inactive.workers = n.inactive.workers[:0]
 
+	for _, p := range n.inactive.providernodes {
+
+		// TODO setup config with options
+		err := p.Deps.PopulateRemainingDeps(context.Background(), &cli.Context{}, false)
+		require.NoError(n.t, err)
+
+		shutdownChan := make(chan struct{})
+		taskEngine, err := tasks.StartTasks(ctx, p.Deps)
+		if err != nil {
+			return nil
+		}
+		defer taskEngine.GracefullyTerminate(time.Hour)
+
+		err = rpc.ListenAndServe(ctx, p.Deps, shutdownChan) // Monitor for shutdown.
+		require.NoError(n.t, err)
+		finishCh := node.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
+		//node.ShutdownHandler{Component: "provider", StopFunc: stop},
+
+		<-finishCh
+
+		n.active.providernodes = append(n.active.providernodes, p)
+	}
 	// ---------------------
 	//  MISC
 	// ---------------------
@@ -1067,3 +1130,10 @@ func importPreSealMeta(ctx context.Context, meta genesis.Miner, mds dtypes.Metad
 	size := binary.PutUvarint(buf, uint64(maxSectorID))
 	return mds.Put(ctx, datastore.NewKey(pipeline.StorageCounterDSPrefix), buf[:size])
 }
+
+func envElse(env, els string) string {
+	if v := os.Getenv(env); v != "" {
+		return v
+	}
+	return els
+}

itests/kit/ensemble_presets.go

@@ -101,6 +101,21 @@ func EnsembleOneTwo(t *testing.T, opts ...interface{}) (*TestFullNode, *TestMine
 	return &full, &one, &two, ens
 }
 
+// EnsembleProvider creates and starts an Ensemble with a single full node and a single provider.
+// It does not interconnect nodes nor does it begin mining.
+func EnsembleProvider(t *testing.T, opts ...interface{}) (*TestFullNode, *TestProviderNode, *Ensemble) {
+	opts = append(opts, WithAllSubsystems())
+
+	eopts, nopts := siftOptions(t, opts)
+
+	var (
+		full     TestFullNode
+		provider TestProviderNode
+	)
+	ens := NewEnsemble(t, eopts...).FullNode(&full, nopts...).Provider(&provider, nopts...).Start()
+	return &full, &provider, ens
+}
+
 func siftOptions(t *testing.T, opts []interface{}) (eopts []EnsembleOpt, nopts []NodeOpt) {
 	for _, v := range opts {
 		switch o := v.(type) {

itests/kit/node_full.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet/key"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/cmd/lotus-provider/deps"
 	"github.com/filecoin-project/lotus/gateway"
 	"github.com/filecoin-project/lotus/node"
 )
@@ -54,6 +55,17 @@ type TestFullNode struct {
 	options nodeOpts
 }
 
+// TestProviderNode represents a Provider node enrolled in an Ensemble.
+type TestProviderNode struct {
+	v1api.LotusProviderStruct
+
+	t *testing.T
+
+	*deps.Deps
+
+	options nodeOpts
+}
+
 func MergeFullNodes(fullNodes []*TestFullNode) *TestFullNode {
 	var wrappedFullNode TestFullNode
 	var fns api.FullNodeStruct

itests/kit/node_miner.go

@@ -37,6 +37,8 @@ const (
 	SSealing
 	SSectorStorage
 
+	SHarmony
+
 	MinerSubsystems = iota
 )
 

itests/kit/node_opts.go

@@ -89,6 +89,13 @@ func WithAllSubsystems() NodeOpt {
 	}
 }
 
+func WithSectorIndexDB() NodeOpt {
+	return func(opts *nodeOpts) error {
+		opts.subsystems = opts.subsystems.Add(SHarmony)
+		return nil
+	}
+}
+
 func WithSubsystems(systems ...MinerSubsystem) NodeOpt {
 	return func(opts *nodeOpts) error {
 		for _, s := range systems {

itests/path_type_filters_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 func TestPathTypeFilters(t *testing.T) {
+
 	runTest := func(t *testing.T, name string, asserts func(t *testing.T, ctx context.Context, miner *kit.TestMiner, run func())) {
 		t.Run(name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())

itests/raft_messagesigner_test.go

@@ -1,577 +0,0 @@
-package itests
-
-import (
-	"context"
-	"crypto/rand"
-	"fmt"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	gorpc "github.com/libp2p/go-libp2p-gorpc"
-	libp2pcrypto "github.com/libp2p/go-libp2p/core/crypto"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/exitcode"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagesigner"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/itests/kit"
-	consensus "github.com/filecoin-project/lotus/lib/consensus/raft"
-	"github.com/filecoin-project/lotus/node"
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/impl"
-	"github.com/filecoin-project/lotus/node/modules"
-)
-
-func generatePrivKey() (*kit.Libp2p, error) {
-	privkey, _, err := libp2pcrypto.GenerateEd25519Key(rand.Reader)
-	if err != nil {
-		return nil, err
-	}
-
-	peerId, err := peer.IDFromPrivateKey(privkey)
-	if err != nil {
-		return nil, err
-	}
-
-	return &kit.Libp2p{PeerID: peerId, PrivKey: privkey}, nil
-}
-
-func getRaftState(ctx context.Context, t *testing.T, node *kit.TestFullNode) *api.RaftStateData {
-	raftState, err := node.RaftState(ctx)
-	require.NoError(t, err)
-	return raftState
-}
-
-func setup(ctx context.Context, t *testing.T, node0 *kit.TestFullNode, node1 *kit.TestFullNode, node2 *kit.TestFullNode, miner *kit.TestMiner) *kit.Ensemble {
-
-	blockTime := 1 * time.Second
-
-	pkey0, _ := generatePrivKey()
-	pkey1, _ := generatePrivKey()
-	pkey2, _ := generatePrivKey()
-
-	pkeys := []*kit.Libp2p{pkey0, pkey1, pkey2}
-	initPeerSet := []string{}
-	for _, pkey := range pkeys {
-		initPeerSet = append(initPeerSet, "/p2p/"+pkey.PeerID.String())
-	}
-
-	//initPeerSet := []peer.ID{pkey0.PeerID, pkey1.PeerID, pkey2.PeerID}
-
-	raftOps := kit.ConstructorOpts(
-		node.Override(new(*gorpc.Client), modules.NewRPCClient),
-		node.Override(new(*consensus.ClusterRaftConfig), func() *consensus.ClusterRaftConfig {
-			cfg := consensus.DefaultClusterRaftConfig()
-			cfg.InitPeerset = initPeerSet
-			return cfg
-		}),
-		node.Override(new(*consensus.Consensus), consensus.NewConsensusWithRPCClient(false)),
-		node.Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
-		node.Override(new(messagesigner.MsgSigner), func(ms *messagesigner.MessageSignerConsensus) *messagesigner.MessageSignerConsensus { return ms }),
-		node.Override(new(*modules.RPCHandler), modules.NewRPCHandler),
-		node.Override(node.GoRPCServer, modules.NewRPCServer),
-	)
-	//raftOps := kit.ConstructorOpts()
-
-	ens := kit.NewEnsemble(t).FullNode(node0, raftOps, kit.ThroughRPC()).FullNode(node1, raftOps, kit.ThroughRPC()).FullNode(node2, raftOps, kit.ThroughRPC())
-	node0.AssignPrivKey(pkey0)
-	node1.AssignPrivKey(pkey1)
-	node2.AssignPrivKey(pkey2)
-
-	nodes := []*kit.TestFullNode{node0, node1, node2}
-	wrappedFullNode := kit.MergeFullNodes(nodes)
-
-	ens.MinerEnroll(miner, wrappedFullNode, kit.WithAllSubsystems(), kit.ThroughRPC())
-	ens.Start()
-
-	// Import miner wallet to all nodes
-	addr0, err := node0.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr1, err := node1.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr2, err := node2.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-
-	fmt.Println(addr0, addr1, addr2)
-
-	ens.InterconnectAll()
-
-	ens.AddInactiveMiner(miner)
-	ens.Start()
-
-	ens.InterconnectAll().BeginMining(blockTime)
-
-	return ens
-}
-
-func TestRaftState(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	fmt.Println(node0.WalletList(context.Background()))
-	fmt.Println(node1.WalletList(context.Background()))
-	fmt.Println(node2.WalletList(context.Background()))
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-
-	mu := uuid.New()
-	smHalfBal, err := node0.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.EqualValues(t, rstate0, rstate1)
-	require.EqualValues(t, rstate0, rstate2)
-}
-
-func TestRaftStateLeaderDisconnects(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := node0.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.True(t, reflect.DeepEqual(rstate0, rstate1))
-	require.True(t, reflect.DeepEqual(rstate0, rstate2))
-
-	leader, err := node1.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	msg2 := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    leaderNode.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	mu2 := uuid.New()
-	signedMsg2, err := leaderNode.MpoolPushMessage(ctx, msg2, &api.MessageSendSpec{
-		MsgUuid: mu2,
-	})
-	require.NoError(t, err)
-	mLookup, err = leaderNode.StateWaitMsg(ctx, signedMsg2.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-func TestRaftStateMiner(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	fmt.Println(node0.WalletList(context.Background()))
-	fmt.Println(node1.WalletList(context.Background()))
-	fmt.Println(node2.WalletList(context.Background()))
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-	mLookup, err := node0.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate0 := getRaftState(ctx, t, &node0)
-	rstate1 := getRaftState(ctx, t, &node1)
-	rstate2 := getRaftState(ctx, t, &node2)
-
-	require.EqualValues(t, rstate0, rstate1)
-	require.EqualValues(t, rstate0, rstate2)
-}
-
-func TestRaftStateLeaderDisconnectsMiner(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	// Take leader node down
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	msg2 := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	mu2 := uuid.New()
-
-	signedMsg2, err := miner.FullNode.MpoolPushMessage(ctx, msg2, &api.MessageSendSpec{
-		MaxFee:  abi.TokenAmount(config.DefaultDefaultMaxFee),
-		MsgUuid: mu2,
-	})
-	require.NoError(t, err)
-
-	mLookup, err := leaderNode.StateWaitMsg(ctx, signedMsg2.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-// Miner sends message on leader
-// Leader disconnects
-// Call StateWaitMsg on new leader
-func TestLeaderDisconnectsCheckMsgStateOnNewLeader(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	// Take leader node down
-	err = leaderNode.Stop(ctx)
-	require.NoError(t, err)
-	oldLeaderNode := leaderNode
-
-	time.Sleep(5 * time.Second)
-
-	// Check if all active nodes update their leader
-	newLeader := leader
-	for _, n := range nodes {
-		if n != leaderNode {
-			newLeader, err = n.RaftLeader(ctx)
-			require.NoError(t, err)
-			require.NotEqual(t, newLeader, leader)
-		}
-	}
-
-	require.NotEqual(t, newLeader, leader)
-	leaderNode = peerToNode[newLeader]
-
-	mLookup, err := leaderNode.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-	require.NoError(t, err)
-	require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-
-	rstate := getRaftState(ctx, t, leaderNode)
-
-	// Check if Raft state is consistent on all active nodes
-	for _, n := range nodes {
-		if n != oldLeaderNode {
-			rs := getRaftState(ctx, t, n)
-			require.True(t, reflect.DeepEqual(rs, rstate))
-		}
-	}
-}
-
-func TestChainStoreSync(t *testing.T) {
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-
-	setup(ctx, t, &node0, &node1, &node2, &miner)
-
-	peerToNode := make(map[peer.ID]*kit.TestFullNode)
-	for _, n := range nodes {
-		peerToNode[n.Pkey.PeerID] = n
-	}
-
-	bal, err := node0.WalletBalance(ctx, node0.DefaultKey.Address)
-	require.NoError(t, err)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-	leaderNode := peerToNode[leader]
-
-	msgHalfBal := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.Div(bal, big.NewInt(2)),
-	}
-	mu := uuid.New()
-	smHalfBal, err := miner.FullNode.MpoolPushMessage(ctx, msgHalfBal, &api.MessageSendSpec{
-		MsgUuid: mu,
-	})
-	require.NoError(t, err)
-
-	for _, n := range nodes {
-		fmt.Println(n != leaderNode)
-		if n != leaderNode {
-			mLookup, err := n.StateWaitMsg(ctx, smHalfBal.Cid(), 3, api.LookbackNoLimit, true)
-			require.NoError(t, err)
-			require.Equal(t, exitcode.Ok, mLookup.Receipt.ExitCode)
-			//break
-		}
-	}
-}
-
-func TestGoRPCAuth(t *testing.T) {
-	// TODO Fix Raft, then enable this test. https://github.com/filecoin-project/lotus/issues/9888
-	t.SkipNow()
-
-	blockTime := 1 * time.Second
-
-	kit.QuietMiningLogs()
-	ctx := context.Background()
-
-	var (
-		node0 kit.TestFullNode
-		node1 kit.TestFullNode
-		node2 kit.TestFullNode
-		node3 kit.TestFullNode
-		miner kit.TestMiner
-	)
-
-	pkey0, _ := generatePrivKey()
-	pkey1, _ := generatePrivKey()
-	pkey2, _ := generatePrivKey()
-
-	pkeys := []*kit.Libp2p{pkey0, pkey1, pkey2}
-	initPeerSet := []string{}
-	for _, pkey := range pkeys {
-		initPeerSet = append(initPeerSet, "/p2p/"+pkey.PeerID.String())
-	}
-
-	raftOps := kit.ConstructorOpts(
-		node.Override(new(*gorpc.Client), modules.NewRPCClient),
-		node.Override(new(*consensus.ClusterRaftConfig), func() *consensus.ClusterRaftConfig {
-			cfg := consensus.DefaultClusterRaftConfig()
-			cfg.InitPeerset = initPeerSet
-			return cfg
-		}),
-		node.Override(new(*consensus.Consensus), consensus.NewConsensusWithRPCClient(false)),
-		node.Override(new(*messagesigner.MessageSignerConsensus), messagesigner.NewMessageSignerConsensus),
-		node.Override(new(messagesigner.MsgSigner), func(ms *messagesigner.MessageSignerConsensus) *messagesigner.MessageSignerConsensus { return ms }),
-		node.Override(new(*modules.RPCHandler), modules.NewRPCHandler),
-		node.Override(node.GoRPCServer, modules.NewRPCServer),
-	)
-	//raftOps := kit.ConstructorOpts()
-
-	ens := kit.NewEnsemble(t).FullNode(&node0, raftOps, kit.ThroughRPC()).FullNode(&node1, raftOps, kit.ThroughRPC()).FullNode(&node2, raftOps, kit.ThroughRPC()).FullNode(&node3, raftOps)
-	node0.AssignPrivKey(pkey0)
-	node1.AssignPrivKey(pkey1)
-	node2.AssignPrivKey(pkey2)
-
-	nodes := []*kit.TestFullNode{&node0, &node1, &node2}
-	wrappedFullNode := kit.MergeFullNodes(nodes)
-
-	ens.MinerEnroll(&miner, wrappedFullNode, kit.WithAllSubsystems(), kit.ThroughRPC())
-	ens.Start()
-
-	// Import miner wallet to all nodes
-	addr0, err := node0.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr1, err := node1.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-	addr2, err := node2.WalletImport(ctx, &miner.OwnerKey.KeyInfo)
-	require.NoError(t, err)
-
-	fmt.Println(addr0, addr1, addr2)
-
-	ens.InterconnectAll()
-
-	ens.AddInactiveMiner(&miner)
-	ens.Start()
-
-	ens.InterconnectAll().BeginMining(blockTime)
-
-	leader, err := node0.RaftLeader(ctx)
-	require.NoError(t, err)
-
-	client := node3.FullNode.(*impl.FullNodeAPI).RaftAPI.MessageSigner.Consensus.RpcClient
-	method := "MpoolPushMessage"
-
-	msg := &types.Message{
-		From:  miner.OwnerKey.Address,
-		To:    node0.DefaultKey.Address,
-		Value: big.NewInt(100000),
-	}
-	msgWhole := &api.MpoolMessageWhole{Msg: msg}
-	var ret types.SignedMessage
-
-	err = client.CallContext(ctx, leader, "Consensus", method, msgWhole, &ret)
-	require.True(t, gorpc.IsAuthorizationError(err))
-
-}

itests/wdpost_worker_config_test.go

@@ -139,7 +139,7 @@ func TestWindowPostNoBuiltinWindowWithWorker(t *testing.T) {
 
 	t.Log("post message landed")
 
-	bm.MineBlocks(ctx, 2*time.Millisecond)
+	bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
 
 	waitUntil = di.Open + di.WPoStChallengeWindow*3
 	t.Logf("End for head.Height > %d", waitUntil)

itests/worker_test.go

@@ -146,7 +146,7 @@ func TestWindowPostWorker(t *testing.T) {
 	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 	require.NoError(t, err)
 
-	bm := ens.InterconnectAll().BeginMining(2 * time.Millisecond)[0]
+	bm := ens.InterconnectAll().BeginMiningMustPost(2 * time.Millisecond)[0]
 
 	di = di.NextNotElapsed()
 
@@ -172,7 +172,7 @@ func TestWindowPostWorker(t *testing.T) {
 
 	t.Log("post message landed")
 
-	bm.MineBlocks(ctx, 2*time.Millisecond)
+	bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
 
 	waitUntil = di.Open + di.WPoStChallengeWindow*3
 	t.Logf("End for head.Height > %d", waitUntil)
@@ -235,6 +235,8 @@ func TestWindowPostWorker(t *testing.T) {
 type badWorkerStorage struct {
 	paths.Store
 
+	t *testing.T
+
 	badsector   *uint64
 	notBadCount int
 }
@@ -242,10 +244,12 @@ type badWorkerStorage struct {
 func (bs *badWorkerStorage) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error) {
 	if atomic.LoadUint64(bs.badsector) == uint64(si.SectorNumber) {
 		bs.notBadCount--
+		bs.t.Logf("Generating proof for sector %d maybe bad nbc=%d", si.SectorNumber, bs.notBadCount)
 		if bs.notBadCount < 0 {
 			return nil, xerrors.New("no proof for you")
 		}
 	}
+	bs.t.Logf("Generating proof for sector %d", si.SectorNumber)
 	return bs.Store.GenerateSingleVanillaProof(ctx, minerID, si, ppt)
 }
 
@@ -268,6 +272,7 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) {
 			return &badWorkerStorage{
 				Store:     store,
 				badsector: &badsector,
+				t:         t,
 			}
 		}),
 		kit.ConstructorOpts(node.ApplyIf(node.IsType(repo.StorageMiner),
@@ -275,6 +280,7 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) {
 				return &badWorkerStorage{
 					Store:       store,
 					badsector:   &badsector,
+					t:           t,
 					notBadCount: 1,
 				}
 			}))))
@@ -506,157 +512,6 @@ func TestWorkerName(t *testing.T) {
 	require.True(t, found)
 }
 
-// Tests that V1_1 proofs on post workers with faults
-func TestWindowPostV1P1NV20WorkerFault(t *testing.T) {
-	kit.QuietMiningLogs()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	blocktime := 2 * time.Millisecond
-
-	sectors := 2 * 48 * 2
-	var badsector uint64 = 100000
-
-	client, miner, _, ens := kit.EnsembleWorker(t,
-		kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
-		kit.GenesisNetworkVersion(network.Version20),
-		kit.ConstructorOpts(
-			node.Override(new(config.ProvingConfig), func() config.ProvingConfig {
-				c := config.DefaultStorageMiner()
-				c.Proving.DisableBuiltinWindowPoSt = true
-				return c.Proving
-			}),
-			node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
-				config.DefaultStorageMiner().Fees,
-				config.ProvingConfig{
-					DisableBuiltinWindowPoSt:  true,
-					DisableBuiltinWinningPoSt: false,
-					DisableWDPoStPreChecks:    false,
-				},
-			)),
-			node.Override(new(paths.Store), func(store *paths.Remote) paths.Store {
-				return &badWorkerStorage{
-					Store:       store,
-					badsector:   &badsector,
-					notBadCount: 1,
-				}
-			})),
-		kit.ThroughRPC(),
-		kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}),
-		kit.WithWorkerStorage(func(store paths.Store) paths.Store {
-			return &badWorkerStorage{
-				Store:     store,
-				badsector: &badsector,
-			}
-		}))
-
-	bm := ens.InterconnectAll().BeginMining(blocktime)[0]
-
-	maddr, err := miner.ActorAddress(ctx)
-	require.NoError(t, err)
-
-	// wait for sectors to be committed
-	require.Eventually(t, func() bool {
-		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-		require.NoError(t, err)
-
-		parts, err := client.StateMinerPartitions(ctx, maddr, di.Index, types.EmptyTSK)
-		require.NoError(t, err)
-
-		return len(parts) > 1
-	}, 30*time.Second, 100*time.Millisecond)
-
-	// Wait until just before a deadline opens
-	{
-		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-		require.NoError(t, err)
-
-		di = di.NextNotElapsed()
-
-		t.Log("Running one proving period")
-		waitUntil := di.Open + di.WPoStChallengeWindow - di.WPoStChallengeLookback - 1
-		client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
-
-		t.Log("Waiting for post message")
-		bm.Stop()
-	}
-
-	// Remove one sector in the next deadline (so it's skipped)
-	{
-		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
-		require.NoError(t, err)
-
-		parts, err := client.StateMinerPartitions(ctx, maddr, di.Index+1, types.EmptyTSK)
-		require.NoError(t, err)
-		require.Greater(t, len(parts), 0)
-
-		secs := parts[0].AllSectors
-		n, err := secs.Count()
-		require.NoError(t, err)
-		require.Equal(t, uint64(2), n)
-
-		// Drop the sector in first partition
-		sid, err := secs.First()
-		require.NoError(t, err)
-
-		t.Logf("Drop sector %d; dl %d part %d", sid, di.Index, 0)
-
-		atomic.StoreUint64(&badsector, sid)
-		require.NoError(t, err)
-	}
-
-	bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
-
-	mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK)
-	require.NoError(t, err)
-
-	wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
-	require.NoError(t, err)
-	en := wact.Nonce
-
-	// wait for a new message to be sent from worker address, it will be a PoSt
-
-waitForProof:
-	for {
-		//stm: @CHAIN_STATE_GET_ACTOR_001
-		wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
-		require.NoError(t, err)
-		if wact.Nonce > en {
-			break waitForProof
-		}
-
-		build.Clock.Sleep(blocktime)
-	}
-
-	slm, err := client.StateListMessages(ctx, &api.MessageMatch{To: maddr}, types.EmptyTSK, 0)
-	require.NoError(t, err)
-
-	pmr, err := client.StateSearchMsg(ctx, types.EmptyTSK, slm[0], -1, false)
-	require.NoError(t, err)
-
-	nv, err := client.StateNetworkVersion(ctx, pmr.TipSet)
-	require.NoError(t, err)
-	require.Equal(t, network.Version20, nv)
-
-	require.True(t, pmr.Receipt.ExitCode.IsSuccess())
-
-	slmsg, err := client.ChainGetMessage(ctx, slm[0])
-	require.NoError(t, err)
-
-	var params miner11.SubmitWindowedPoStParams
-	require.NoError(t, params.UnmarshalCBOR(bytes.NewBuffer(slmsg.Params)))
-	require.Equal(t, abi.RegisteredPoStProof_StackedDrgWindow2KiBV1_1, params.Proofs[0].PoStProof)
-
-	require.Len(t, params.Partitions, 2)
-	sc0, err := params.Partitions[0].Skipped.Count()
-	require.NoError(t, err)
-	require.Equal(t, uint64(1), sc0)
-	sc1, err := params.Partitions[1].Skipped.Count()
-	require.NoError(t, err)
-	require.Equal(t, uint64(0), sc1)
-}
-
 // Tests that V1_1 proofs on post worker
 func TestWindowPostV1P1NV20Worker(t *testing.T) {
 	kit.QuietMiningLogs()
@@ -685,7 +540,7 @@ func TestWindowPostV1P1NV20Worker(t *testing.T) {
 		kit.ThroughRPC(),
 		kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}))
 
-	ens.InterconnectAll().BeginMining(blocktime)
+	ens.InterconnectAll().BeginMiningMustPost(blocktime)
 
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)

journal/fsjournal/fs.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build"
@@ -37,7 +38,16 @@ type fsJournal struct {
 // OpenFSJournal constructs a rolling filesystem journal, with a default
 // per-file size limit of 1GiB.
 func OpenFSJournal(lr repo.LockedRepo, disabled journal.DisabledEvents) (journal.Journal, error) {
-	dir := filepath.Join(lr.Path(), "journal")
+	return OpenFSJournalPath(lr.Path(), disabled)
+}
+
+func OpenFSJournalPath(path string, disabled journal.DisabledEvents) (journal.Journal, error) {
+	path, err := homedir.Expand(path)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to expand repo path: %w", err)
+	}
+
+	dir := filepath.Join(path, "journal")
 	if err := os.MkdirAll(dir, 0755); err != nil {
 		return nil, fmt.Errorf("failed to mk directory %s for file journal: %w", dir, err)
 	}

journal/types.go

@@ -26,6 +26,9 @@ func ParseDisabledEvents(s string) (DisabledEvents, error) {
 	s = strings.TrimSpace(s) // sanitize
 	evts := strings.Split(s, ",")
 	ret := make(DisabledEvents, 0, len(evts))
+	if len(s) == 0 {
+		return ret, nil
+	}
 	for _, evt := range evts {
 		evt = strings.TrimSpace(evt) // sanitize
 		s := strings.Split(evt, ":")

lib/consensus/raft/config.go

@@ -1,135 +0,0 @@
-package consensus
-
-import (
-	"io"
-	"path/filepath"
-	"time"
-
-	hraft "github.com/hashicorp/raft"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/node/config"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-// Configuration defaults
-var (
-	DefaultDataSubFolder        = "raft-cluster"
-	DefaultWaitForLeaderTimeout = 15 * time.Second
-	DefaultCommitRetries        = 1
-	DefaultNetworkTimeout       = 100 * time.Second
-	DefaultCommitRetryDelay     = 200 * time.Millisecond
-	DefaultBackupsRotate        = 6
-)
-
-// ClusterRaftConfig allows to configure the Raft Consensus component for the node cluster.
-type ClusterRaftConfig struct {
-	// config to enabled node cluster with raft consensus
-	ClusterModeEnabled bool
-	// A folder to store Raft's data.
-	DataFolder string
-	// InitPeerset provides the list of initial cluster peers for new Raft
-	// peers (with no prior state). It is ignored when Raft was already
-	// initialized or when starting in staging mode.
-	InitPeerset []string
-	// LeaderTimeout specifies how long to wait for a leader before
-	// failing an operation.
-	WaitForLeaderTimeout time.Duration
-	// NetworkTimeout specifies how long before a Raft network
-	// operation is timed out
-	NetworkTimeout time.Duration
-	// CommitRetries specifies how many times we retry a failed commit until
-	// we give up.
-	CommitRetries int
-	// How long to wait between retries
-	CommitRetryDelay time.Duration
-	// BackupsRotate specifies the maximum number of Raft's DataFolder
-	// copies that we keep as backups (renaming) after cleanup.
-	BackupsRotate int
-	// A Hashicorp Raft's configuration object.
-	RaftConfig *hraft.Config
-
-	// Tracing enables propagation of contexts across binary boundaries.
-	Tracing bool
-}
-
-func DefaultClusterRaftConfig() *ClusterRaftConfig {
-	var cfg ClusterRaftConfig
-	cfg.DataFolder = "" // empty so it gets omitted
-	cfg.InitPeerset = []string{}
-	cfg.WaitForLeaderTimeout = DefaultWaitForLeaderTimeout
-	cfg.NetworkTimeout = DefaultNetworkTimeout
-	cfg.CommitRetries = DefaultCommitRetries
-	cfg.CommitRetryDelay = DefaultCommitRetryDelay
-	cfg.BackupsRotate = DefaultBackupsRotate
-	cfg.RaftConfig = hraft.DefaultConfig()
-
-	// These options are imposed over any Default Raft Config.
-	cfg.RaftConfig.ShutdownOnRemove = false
-	cfg.RaftConfig.LocalID = "will_be_set_automatically"
-
-	// Set up logging
-	cfg.RaftConfig.LogOutput = io.Discard
-	return &cfg
-}
-
-func NewClusterRaftConfig(userRaftConfig *config.UserRaftConfig) *ClusterRaftConfig {
-	var cfg ClusterRaftConfig
-	cfg.DataFolder = userRaftConfig.DataFolder
-	cfg.InitPeerset = userRaftConfig.InitPeersetMultiAddr
-	cfg.WaitForLeaderTimeout = time.Duration(userRaftConfig.WaitForLeaderTimeout)
-	cfg.NetworkTimeout = time.Duration(userRaftConfig.NetworkTimeout)
-	cfg.CommitRetries = userRaftConfig.CommitRetries
-	cfg.CommitRetryDelay = time.Duration(userRaftConfig.CommitRetryDelay)
-	cfg.BackupsRotate = userRaftConfig.BackupsRotate
-
-	// Keep this to be default hraft config for now
-	cfg.RaftConfig = hraft.DefaultConfig()
-
-	// These options are imposed over any Default Raft Config.
-	cfg.RaftConfig.ShutdownOnRemove = false
-	cfg.RaftConfig.LocalID = "will_be_set_automatically"
-
-	// Set up logging
-	cfg.RaftConfig.LogOutput = io.Discard
-
-	return &cfg
-
-}
-
-// Validate checks that this configuration has working values,
-// at least in appearance.
-func ValidateConfig(cfg *ClusterRaftConfig) error {
-	if cfg.RaftConfig == nil {
-		return xerrors.Errorf("no hashicorp/raft.Config")
-	}
-	if cfg.WaitForLeaderTimeout <= 0 {
-		return xerrors.Errorf("wait_for_leader_timeout <= 0")
-	}
-
-	if cfg.NetworkTimeout <= 0 {
-		return xerrors.Errorf("network_timeout <= 0")
-	}
-
-	if cfg.CommitRetries < 0 {
-		return xerrors.Errorf("commit_retries is invalid")
-	}
-
-	if cfg.CommitRetryDelay <= 0 {
-		return xerrors.Errorf("commit_retry_delay is invalid")
-	}
-
-	if cfg.BackupsRotate <= 0 {
-		return xerrors.Errorf("backups_rotate should be larger than 0")
-	}
-
-	return hraft.ValidateConfig(cfg.RaftConfig)
-}
-
-// GetDataFolder returns the Raft data folder that we are using.
-func (cfg *ClusterRaftConfig) GetDataFolder(repo repo.LockedRepo) string {
-	if cfg.DataFolder == "" {
-		return filepath.Join(repo.Path(), DefaultDataSubFolder)
-	}
-	return filepath.Join(repo.Path(), cfg.DataFolder)
-}

lib/consensus/raft/consensus.go

@@ -1,512 +0,0 @@
-// Package raft implements a Consensus component for IPFS Cluster which uses
-// Raft (go-libp2p-raft).
-package consensus
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"sort"
-	"time"
-
-	"github.com/google/uuid"
-	"golang.org/x/exp/slices"
-
-	addr "github.com/filecoin-project/go-address"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagepool"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/addrutil"
-	"github.com/filecoin-project/lotus/node/repo"
-
-	//ds "github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log/v2"
-	consensus "github.com/libp2p/go-libp2p-consensus"
-	rpc "github.com/libp2p/go-libp2p-gorpc"
-	libp2praft "github.com/libp2p/go-libp2p-raft"
-	host "github.com/libp2p/go-libp2p/core/host"
-	peer "github.com/libp2p/go-libp2p/core/peer"
-)
-
-var logger = logging.Logger("raft")
-
-type RaftState struct {
-	NonceMap api.NonceMapType
-	MsgUuids api.MsgUuidMapType
-
-	// TODO: add comment explaining why this is needed
-	// We need a reference to the messagepool in the raft state in order to
-	// sync messages that have been sent by the leader node
-	// Miner calls StateWaitMsg after MpoolPushMessage to check if the message has
-	// landed on chain. This check requires the message be stored in the local chainstore
-	// If a leadernode goes down after sending a message to the chain and is replaced by
-	// another node, the other node needs to have this message in its chainstore for the
-	// above check to succeed.
-
-	// This is because the miner only stores signed CIDs but the message received from in a
-	// block will be unsigned (for BLS). Hence, the process relies on the node to store the
-	// signed message which holds a copy of the unsigned message to properly perform all the
-	// needed checks
-	Mpool *messagepool.MessagePool
-}
-
-func newRaftState(mpool *messagepool.MessagePool) *RaftState {
-	return &RaftState{
-		NonceMap: make(map[addr.Address]uint64),
-		MsgUuids: make(map[uuid.UUID]*types.SignedMessage),
-		Mpool:    mpool,
-	}
-}
-
-type ConsensusOp struct {
-	Nonce     uint64               `codec:"nonce,omitempty"`
-	Uuid      uuid.UUID            `codec:"uuid,omitempty"`
-	Addr      addr.Address         `codec:"addr,omitempty"`
-	SignedMsg *types.SignedMessage `codec:"signedMsg,omitempty"`
-}
-
-func (c ConsensusOp) ApplyTo(state consensus.State) (consensus.State, error) {
-	s := state.(*RaftState)
-	s.NonceMap[c.Addr] = c.Nonce
-	if c.SignedMsg != nil {
-
-		// Deep copy to tmp
-		var buffer bytes.Buffer
-		err := c.SignedMsg.MarshalCBOR(&buffer)
-		if err != nil {
-			return nil, err
-		}
-		tmp, err := types.DecodeSignedMessage(buffer.Bytes())
-		if err != nil {
-			return nil, err
-		}
-		s.MsgUuids[c.Uuid] = tmp
-
-		_, err = s.Mpool.Push(context.TODO(), tmp, false)
-		// Since this is only meant to keep messages in sync, ignore any error which
-		// shows the message already exists in the mpool
-		if err != nil && !api.ErrorIsIn(err, []error{messagepool.ErrExistingNonce}) {
-			return nil, err
-		}
-	}
-
-	return s, nil
-}
-
-var _ consensus.Op = &ConsensusOp{}
-
-// Consensus handles the work of keeping a shared-state between
-// the peers of a Lotus Cluster, as well as modifying that state and
-// applying any updates in a thread-safe manner.
-type Consensus struct {
-	ctx    context.Context
-	cancel func()
-	config *ClusterRaftConfig
-
-	host host.Host
-
-	consensus consensus.OpLogConsensus
-	actor     consensus.Actor
-	raft      *raftWrapper
-	state     *RaftState
-
-	RpcClient *rpc.Client
-	rpcReady  chan struct{}
-	readyCh   chan struct{}
-
-	peerSet []peer.ID
-	repo    repo.LockedRepo
-}
-
-// NewConsensus builds a new ClusterConsensus component using Raft.
-//
-// Raft saves state snapshots regularly and persists log data in a bolt
-// datastore. Therefore, unless memory usage is a concern, it is recommended
-// to use an in-memory go-datastore as store parameter.
-//
-// The staging parameter controls if the Raft peer should start in
-// staging mode (used when joining a new Raft peerset with other peers).
-func NewConsensus(host host.Host, cfg *ClusterRaftConfig, mpool *messagepool.MessagePool, repo repo.LockedRepo, staging bool) (*Consensus, error) {
-	err := ValidateConfig(cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	logger.Debug("starting Consensus and waiting for a leader...")
-	state := newRaftState(mpool)
-
-	consensus := libp2praft.NewOpLog(state, &ConsensusOp{})
-
-	raft, err := newRaftWrapper(host, cfg, consensus.FSM(), repo, staging)
-	if err != nil {
-		logger.Error("error creating raft: ", err)
-		cancel()
-		return nil, err
-	}
-	actor := libp2praft.NewActor(raft.raft)
-	consensus.SetActor(actor)
-
-	peers := []peer.ID{}
-	addrInfos, err := addrutil.ParseAddresses(ctx, cfg.InitPeerset)
-	if err != nil {
-		logger.Error("error parsing addresses: ", err)
-		cancel()
-		return nil, err
-	}
-
-	for _, addrInfo := range addrInfos {
-		peers = append(peers, addrInfo.ID)
-
-		// Add peer to address book
-		host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, time.Hour*100)
-	}
-
-	cc := &Consensus{
-		ctx:       ctx,
-		cancel:    cancel,
-		config:    cfg,
-		host:      host,
-		consensus: consensus,
-		actor:     actor,
-		state:     state,
-		raft:      raft,
-		peerSet:   peers,
-		rpcReady:  make(chan struct{}, 1),
-		readyCh:   make(chan struct{}, 1),
-		repo:      repo,
-	}
-
-	go cc.finishBootstrap()
-	return cc, nil
-
-}
-
-// TODO: Merge with NewConsensus and remove the rpcReady chan
-func NewConsensusWithRPCClient(staging bool) func(host host.Host,
-	cfg *ClusterRaftConfig,
-	rpcClient *rpc.Client,
-	mpool *messagepool.MessagePool,
-	repo repo.LockedRepo,
-) (*Consensus, error) {
-
-	return func(host host.Host, cfg *ClusterRaftConfig, rpcClient *rpc.Client, mpool *messagepool.MessagePool, repo repo.LockedRepo) (*Consensus, error) {
-		cc, err := NewConsensus(host, cfg, mpool, repo, staging)
-		if err != nil {
-			return nil, err
-		}
-		cc.RpcClient = rpcClient
-		cc.rpcReady <- struct{}{}
-		return cc, nil
-	}
-}
-
-// WaitForSync waits for a leader and for the state to be up to date, then returns.
-func (cc *Consensus) WaitForSync(ctx context.Context) error {
-
-	leaderCtx, cancel := context.WithTimeout(ctx, cc.config.WaitForLeaderTimeout)
-	defer cancel()
-
-	// 1 - wait for leader
-	// 2 - wait until we are a Voter
-	// 3 - wait until last index is applied
-
-	// From raft docs:
-
-	// once a staging server receives enough log entries to be sufficiently
-	// caught up to the leader's log, the leader will invoke a  membership
-	// change to change the Staging server to a Voter
-
-	// Thus, waiting to be a Voter is a guarantee that we have a reasonable
-	// up to date state. Otherwise, we might return too early (see
-	// https://github.com/ipfs-cluster/ipfs-cluster/issues/378)
-
-	_, err := cc.raft.WaitForLeader(leaderCtx)
-	if err != nil {
-		return errors.New("error waiting for leader: " + err.Error())
-	}
-
-	err = cc.raft.WaitForVoter(ctx)
-	if err != nil {
-		return errors.New("error waiting to become a Voter: " + err.Error())
-	}
-
-	err = cc.raft.WaitForUpdates(ctx)
-	if err != nil {
-		return errors.New("error waiting for consensus updates: " + err.Error())
-	}
-	return nil
-}
-
-// waits until there is a consensus leader and syncs the state
-// to the tracker. If errors happen, this will return and never
-// signal the component as Ready.
-func (cc *Consensus) finishBootstrap() {
-	// wait until we have RPC to perform any actions.
-	select {
-	case <-cc.ctx.Done():
-		return
-	case <-cc.rpcReady:
-	}
-
-	// Sometimes bootstrap is a no-Op. It only applies when
-	// no state exists and staging=false.
-	_, err := cc.raft.Bootstrap()
-	if err != nil {
-		return
-	}
-
-	logger.Debugf("Bootstrap finished")
-	err = cc.WaitForSync(cc.ctx)
-	if err != nil {
-		return
-	}
-	logger.Debug("Raft state is now up to date")
-	logger.Debug("consensus ready")
-	cc.readyCh <- struct{}{}
-}
-
-// Shutdown stops the component so it will not process any
-// more updates. The underlying consensus is permanently
-// shutdown, along with the libp2p transport.
-func (cc *Consensus) Shutdown(ctx context.Context) error {
-
-	logger.Info("stopping Consensus component")
-
-	// Raft Shutdown
-	err := cc.raft.Shutdown(ctx)
-	if err != nil {
-		logger.Error(err)
-	}
-
-	cc.cancel()
-	close(cc.rpcReady)
-	return nil
-}
-
-// Ready returns a channel which is signaled when the Consensus
-// algorithm has finished bootstrapping and is ready to use
-func (cc *Consensus) Ready(ctx context.Context) <-chan struct{} {
-	return cc.readyCh
-}
-
-// IsTrustedPeer returns true. In Raft we trust all peers.
-func (cc *Consensus) IsTrustedPeer(ctx context.Context, p peer.ID) bool {
-	return slices.Contains(cc.peerSet, p)
-}
-
-// Trust is a no-Op.
-func (cc *Consensus) Trust(ctx context.Context, pid peer.ID) error { return nil }
-
-// Distrust is a no-Op.
-func (cc *Consensus) Distrust(ctx context.Context, pid peer.ID) error { return nil }
-
-// returns true if the operation was redirected to the leader
-// note that if the leader just dissappeared, the rpc call will
-// fail because we haven't heard that it's gone.
-func (cc *Consensus) RedirectToLeader(method string, arg interface{}, ret interface{}) (bool, error) {
-	ctx := cc.ctx
-
-	var finalErr error
-
-	// Retry redirects
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("redirect try %d", i)
-		leader, err := cc.Leader(ctx)
-
-		// No leader, wait for one
-		if err != nil {
-			logger.Warn("there seems to be no leader. Waiting for one")
-			rctx, cancel := context.WithTimeout(ctx, cc.config.WaitForLeaderTimeout)
-			defer cancel()
-			pidstr, err := cc.raft.WaitForLeader(rctx)
-
-			// means we timed out waiting for a leader
-			// we don't retry in this case
-			if err != nil {
-				return false, fmt.Errorf("timed out waiting for leader: %s", err)
-			}
-			leader, err = peer.Decode(pidstr)
-			if err != nil {
-				return false, err
-			}
-		}
-
-		logger.Infof("leader: %s, curr host: %s, peerSet: %s", leader, cc.host.ID(), cc.peerSet)
-
-		// We are the leader. Do not redirect
-		if leader == cc.host.ID() {
-			return false, nil
-		}
-
-		logger.Debugf("redirecting %s to leader: %s", method, leader)
-		finalErr = cc.RpcClient.CallContext(
-			ctx,
-			leader,
-			"Consensus",
-			method,
-			arg,
-			ret,
-		)
-		if finalErr != nil {
-			logger.Errorf("retrying to redirect request to leader: %s", finalErr)
-			time.Sleep(2 * cc.config.RaftConfig.HeartbeatTimeout)
-			continue
-		}
-		break
-	}
-
-	// We tried to redirect, but something happened
-	return true, finalErr
-}
-
-// commit submits a cc.consensus commit. It retries upon failures.
-func (cc *Consensus) Commit(ctx context.Context, op *ConsensusOp) error {
-
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: committing %+v", i, op)
-
-		// this means we are retrying
-		if finalErr != nil {
-			logger.Errorf("retrying upon failed commit (retry %d): %s ",
-				i, finalErr)
-		}
-
-		// Being here means we are the LEADER. We can commit.
-		// now commit the changes to our state
-		_, finalErr = cc.consensus.CommitOp(op)
-		if finalErr != nil {
-			goto RETRY
-		}
-
-	RETRY:
-		time.Sleep(cc.config.CommitRetryDelay)
-	}
-	return finalErr
-}
-
-// AddPeer adds a new peer to participate in this consensus. It will
-// forward the operation to the leader if this is not it.
-func (cc *Consensus) AddPeer(ctx context.Context, pid peer.ID) error {
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: AddPeer %s", i, pid)
-		if finalErr != nil {
-			logger.Errorf("retrying to add peer. Attempt #%d failed: %s", i, finalErr)
-		}
-		ok, err := cc.RedirectToLeader("AddPeer", pid, struct{}{})
-		if err != nil || ok {
-			return err
-		}
-		// Being here means we are the leader and can commit
-		finalErr = cc.raft.AddPeer(ctx, pid)
-		if finalErr != nil {
-			time.Sleep(cc.config.CommitRetryDelay)
-			continue
-		}
-		logger.Infof("peer added to Raft: %s", pid)
-		break
-	}
-	return finalErr
-}
-
-// RmPeer removes a peer from this consensus. It will
-// forward the operation to the leader if this is not it.
-func (cc *Consensus) RmPeer(ctx context.Context, pid peer.ID) error {
-	var finalErr error
-	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: RmPeer %s", i, pid)
-		if finalErr != nil {
-			logger.Errorf("retrying to remove peer. Attempt #%d failed: %s", i, finalErr)
-		}
-		ok, err := cc.RedirectToLeader("RmPeer", pid, struct{}{})
-		if err != nil || ok {
-			return err
-		}
-		// Being here means we are the leader and can commit
-		finalErr = cc.raft.RemovePeer(ctx, pid.String())
-		if finalErr != nil {
-			time.Sleep(cc.config.CommitRetryDelay)
-			continue
-		}
-		logger.Infof("peer removed from Raft: %s", pid)
-		break
-	}
-	return finalErr
-}
-
-// RaftState retrieves the current consensus RaftState. It may error if no RaftState has
-// been agreed upon or the state is not consistent. The returned RaftState is the
-// last agreed-upon RaftState known by this node. No writes are allowed, as all
-// writes to the shared state should happen through the Consensus component
-// methods.
-func (cc *Consensus) State(ctx context.Context) (*RaftState, error) {
-	st, err := cc.consensus.GetLogHead()
-	if err == libp2praft.ErrNoState {
-		return newRaftState(nil), nil
-	}
-
-	if err != nil {
-		return nil, err
-	}
-	state, ok := st.(*RaftState)
-	if !ok {
-		return nil, errors.New("wrong state type")
-	}
-	return state, nil
-}
-
-// Leader returns the peerID of the Leader of the
-// cluster. It returns an error when there is no leader.
-func (cc *Consensus) Leader(ctx context.Context) (peer.ID, error) {
-	// Note the hard-dependency on raft here...
-	raftactor := cc.actor.(*libp2praft.Actor)
-	return raftactor.Leader()
-}
-
-// Clean removes the Raft persisted state.
-func (cc *Consensus) Clean(ctx context.Context) error {
-	//return CleanupRaft(cc.config)
-	return nil
-}
-
-//Rollback replaces the current agreed-upon
-//state with the state provided. Only the consensus leader
-//can perform this operation.
-//func (cc *Consensus) Rollback(state RaftState) error {
-//	// This is unused. It *might* be used for upgrades.
-//	// There is rather untested magic in libp2p-raft's FSM()
-//	// to make this possible.
-//	return cc.consensus.Rollback(state)
-//}
-
-// Peers return the current list of peers in the consensus.
-// The list will be sorted alphabetically.
-func (cc *Consensus) Peers(ctx context.Context) ([]peer.ID, error) {
-
-	peers := []peer.ID{}
-	raftPeers, err := cc.raft.Peers(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("cannot retrieve list of peers: %s", err)
-	}
-
-	sort.Strings(raftPeers)
-
-	for _, p := range raftPeers {
-		id, err := peer.Decode(p)
-		if err != nil {
-			panic("could not decode peer")
-		}
-		peers = append(peers, id)
-	}
-	return peers, nil
-}
-
-func (cc *Consensus) IsLeader(ctx context.Context) bool {
-	leader, _ := cc.Leader(ctx)
-	return leader == cc.host.ID()
-}

lib/consensus/raft/interfaces.go

@@ -1,41 +0,0 @@
-package consensus
-
-import (
-	"context"
-
-	consensus "github.com/libp2p/go-libp2p-consensus"
-	"github.com/libp2p/go-libp2p/core/peer"
-)
-
-type ConsensusAPI interface {
-	// Returns a channel to signal that the consensus layer is ready
-	// allowing the main component to wait for it during start.
-	Ready(context.Context) <-chan struct{}
-
-	AddPeer(context.Context, peer.ID) error
-	RmPeer(context.Context, peer.ID) error
-	State(context.Context) (consensus.State, error)
-	// Provide a node which is responsible to perform
-	// specific tasks which must only run in 1 cluster peer.
-	Leader(context.Context) (peer.ID, error)
-	// Only returns when the consensus state has all log
-	// updates applied to it.
-	WaitForSync(context.Context) error
-	// Clean removes all consensus data.
-	Clean(context.Context) error
-	// Peers returns the peerset participating in the Consensus.
-	Peers(context.Context) ([]peer.ID, error)
-	// IsTrustedPeer returns true if the given peer is "trusted".
-	// This will grant access to more rpc endpoints and a
-	// non-trusted one. This should be fast as it will be
-	// called repeatedly for every remote RPC request.
-	IsTrustedPeer(context.Context, peer.ID) bool
-	// Trust marks a peer as "trusted".
-	Trust(context.Context, peer.ID) error
-	// Distrust removes a peer from the "trusted" set.
-	Distrust(context.Context, peer.ID) error
-	// Returns true if current node is the cluster leader
-	IsLeader(ctx context.Context) bool
-
-	Shutdown(context.Context) error
-}

lib/consensus/raft/raft.go

@@ -1,563 +0,0 @@
-package consensus
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/hashicorp/go-hclog"
-	hraft "github.com/hashicorp/raft"
-	raftboltdb "github.com/hashicorp/raft-boltdb"
-	"github.com/ipfs/go-log/v2"
-	p2praft "github.com/libp2p/go-libp2p-raft"
-	host "github.com/libp2p/go-libp2p/core/host"
-	peer "github.com/libp2p/go-libp2p/core/peer"
-	"go.uber.org/multierr"
-	"go.uber.org/zap"
-
-	"github.com/filecoin-project/lotus/lib/addrutil"
-	"github.com/filecoin-project/lotus/node/repo"
-)
-
-var raftLogger = log.Logger("raft-cluster")
-
-// ErrWaitingForSelf is returned when we are waiting for ourselves to depart
-// the peer set, which won't happen
-var errWaitingForSelf = errors.New("waiting for ourselves to depart")
-
-// RaftMaxSnapshots indicates how many snapshots to keep in the consensus data
-// folder.
-// TODO: Maybe include this in Config. Not sure how useful it is to touch
-// this anyways.
-var RaftMaxSnapshots = 5
-
-// RaftLogCacheSize is the maximum number of logs to cache in-memory.
-// This is used to reduce disk I/O for the recently committed entries.
-var RaftLogCacheSize = 512
-
-// How long we wait for updates during shutdown before snapshotting
-var waitForUpdatesShutdownTimeout = 5 * time.Second
-var waitForUpdatesInterval = 400 * time.Millisecond
-
-// How many times to retry snapshotting when shutting down
-var maxShutdownSnapshotRetries = 5
-
-// raftWrapper wraps the hraft.Raft object and related things like the
-// different stores used or the hraft.Configuration.
-// Its methods provide functionality for working with Raft.
-type raftWrapper struct {
-	ctx           context.Context
-	cancel        context.CancelFunc
-	raft          *hraft.Raft
-	config        *ClusterRaftConfig
-	host          host.Host
-	serverConfig  hraft.Configuration
-	transport     *hraft.NetworkTransport
-	snapshotStore hraft.SnapshotStore
-	logStore      hraft.LogStore
-	stableStore   hraft.StableStore
-	boltdb        *raftboltdb.BoltStore
-	repo          repo.LockedRepo
-	staging       bool
-}
-
-// newRaftWrapper creates a Raft instance and initializes
-// everything leaving it ready to use. Note, that Bootstrap() should be called
-// to make sure the raft instance is usable.
-func newRaftWrapper(
-	host host.Host,
-	cfg *ClusterRaftConfig,
-	fsm hraft.FSM,
-	repo repo.LockedRepo,
-	staging bool,
-) (*raftWrapper, error) {
-
-	raftW := &raftWrapper{}
-	raftW.config = cfg
-	raftW.host = host
-	raftW.staging = staging
-	raftW.repo = repo
-	// Set correct LocalID
-	cfg.RaftConfig.LocalID = hraft.ServerID(host.ID().String())
-
-	df := cfg.GetDataFolder(repo)
-	err := makeDataFolder(df)
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeServerConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeTransport()
-	if err != nil {
-		return nil, err
-	}
-
-	err = raftW.makeStores()
-	if err != nil {
-		return nil, err
-	}
-
-	raftLogger.Debug("creating Raft")
-	raftW.raft, err = hraft.NewRaft(
-		cfg.RaftConfig,
-		fsm,
-		raftW.logStore,
-		raftW.stableStore,
-		raftW.snapshotStore,
-		raftW.transport,
-	)
-	if err != nil {
-		raftLogger.Error("initializing raft: ", err)
-		return nil, err
-	}
-
-	raftW.ctx, raftW.cancel = context.WithCancel(context.Background())
-
-	return raftW, nil
-}
-
-// makeDataFolder creates the folder that is meant to store Raft data. Ensures
-// we always set 0700 mode.
-func makeDataFolder(folder string) error {
-	return os.MkdirAll(folder, 0700)
-}
-
-func (rw *raftWrapper) makeTransport() (err error) {
-	raftLogger.Debug("creating libp2p Raft transport")
-	rw.transport, err = p2praft.NewLibp2pTransport(
-		rw.host,
-		rw.config.NetworkTimeout,
-	)
-	return err
-}
-
-func (rw *raftWrapper) makeStores() error {
-	raftLogger.Debug("creating BoltDB store")
-	df := rw.config.GetDataFolder(rw.repo)
-	store, err := raftboltdb.NewBoltStore(filepath.Join(df, "raft.db"))
-	if err != nil {
-		return err
-	}
-
-	// wraps the store in a LogCache to improve performance.
-	// See consul/agent/consul/server.go
-	cacheStore, err := hraft.NewLogCache(RaftLogCacheSize, store)
-	if err != nil {
-		return err
-	}
-
-	raftLogger.Debug("creating raft snapshot store")
-	snapstore, err := hraft.NewFileSnapshotStoreWithLogger(
-		df,
-		RaftMaxSnapshots,
-		hclog.FromStandardLogger(zap.NewStdLog(log.Logger("raft-snapshot").SugaredLogger.Desugar()), hclog.DefaultOptions),
-	)
-	if err != nil {
-		return err
-	}
-
-	rw.logStore = cacheStore
-	rw.stableStore = store
-	rw.snapshotStore = snapstore
-	rw.boltdb = store
-	return nil
-}
-
-// Bootstrap calls BootstrapCluster on the Raft instance with a valid
-// Configuration (generated from InitPeerset) when Raft has no state
-// and we are not setting up a staging peer. It returns if Raft
-// was boostrapped (true) and an error.
-func (rw *raftWrapper) Bootstrap() (bool, error) {
-	logger.Debug("checking for existing raft states")
-	hasState, err := hraft.HasExistingState(
-		rw.logStore,
-		rw.stableStore,
-		rw.snapshotStore,
-	)
-	if err != nil {
-		return false, err
-	}
-
-	if hasState {
-		logger.Debug("raft cluster is already initialized")
-
-		// Inform the user that we are working with a pre-existing peerset
-		logger.Info("existing Raft state found! raft.InitPeerset will be ignored")
-		cf := rw.raft.GetConfiguration()
-		if err := cf.Error(); err != nil {
-			logger.Debug(err)
-			return false, err
-		}
-		currentCfg := cf.Configuration()
-		srvs := ""
-		for _, s := range currentCfg.Servers {
-			srvs += fmt.Sprintf("        %s\n", s.ID)
-		}
-
-		logger.Debugf("Current Raft Peerset:\n%s\n", srvs)
-		return false, nil
-	}
-
-	if rw.staging {
-		logger.Debug("staging servers do not need initialization")
-		logger.Info("peer is ready to join a cluster")
-		return false, nil
-	}
-
-	voters := ""
-	for _, s := range rw.serverConfig.Servers {
-		voters += fmt.Sprintf("        %s\n", s.ID)
-	}
-
-	logger.Infof("initializing raft cluster with the following voters:\n%s\n", voters)
-
-	future := rw.raft.BootstrapCluster(rw.serverConfig)
-	if err := future.Error(); err != nil {
-		logger.Error("bootstrapping cluster: ", err)
-		return true, err
-	}
-	return true, nil
-}
-
-// create Raft servers configuration. The result is used
-// by Bootstrap() when it proceeds to Bootstrap.
-func (rw *raftWrapper) makeServerConfig() error {
-	peers := []peer.ID{}
-	addrInfos, err := addrutil.ParseAddresses(context.Background(), rw.config.InitPeerset)
-	if err != nil {
-		return err
-	}
-	for _, addrInfo := range addrInfos {
-		peers = append(peers, addrInfo.ID)
-	}
-	rw.serverConfig = makeServerConf(append(peers, rw.host.ID()))
-	return nil
-}
-
-// creates a server configuration with all peers as Voters.
-func makeServerConf(peers []peer.ID) hraft.Configuration {
-	sm := make(map[string]struct{})
-
-	servers := make([]hraft.Server, 0)
-
-	// Servers are peers + self. We avoid duplicate entries below
-	for _, pid := range peers {
-		p := pid.String()
-		_, ok := sm[p]
-		if !ok { // avoid dups
-			sm[p] = struct{}{}
-			servers = append(servers, hraft.Server{
-				Suffrage: hraft.Voter,
-				ID:       hraft.ServerID(p),
-				Address:  hraft.ServerAddress(p),
-			})
-		}
-	}
-	return hraft.Configuration{Servers: servers}
-}
-
-// WaitForLeader holds until Raft says we have a leader.
-// Returns if ctx is canceled.
-func (rw *raftWrapper) WaitForLeader(ctx context.Context) (string, error) {
-	ticker := time.NewTicker(time.Second / 2)
-	for {
-		select {
-		case <-ticker.C:
-			if l := rw.raft.Leader(); l != "" {
-				logger.Debug("waitForleaderTimer")
-				logger.Infof("Current Raft Leader: %s", l)
-				ticker.Stop()
-				return string(l), nil
-			}
-		case <-ctx.Done():
-			return "", ctx.Err()
-		}
-	}
-}
-
-func (rw *raftWrapper) WaitForVoter(ctx context.Context) error {
-	logger.Debug("waiting until we are promoted to a voter")
-
-	pid := hraft.ServerID(rw.host.ID().String())
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			logger.Debugf("%s: get configuration", pid)
-			configFuture := rw.raft.GetConfiguration()
-			if err := configFuture.Error(); err != nil {
-				return err
-			}
-
-			if isVoter(pid, configFuture.Configuration()) {
-				return nil
-			}
-			logger.Debugf("%s: not voter yet", pid)
-
-			time.Sleep(waitForUpdatesInterval)
-		}
-	}
-}
-
-func isVoter(srvID hraft.ServerID, cfg hraft.Configuration) bool {
-	for _, server := range cfg.Servers {
-		if server.ID == srvID && server.Suffrage == hraft.Voter {
-			return true
-		}
-	}
-	return false
-}
-
-// WaitForUpdates holds until Raft has synced to the last index in the log
-func (rw *raftWrapper) WaitForUpdates(ctx context.Context) error {
-
-	logger.Debug("Raft state is catching up to the latest known version. Please wait...")
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			lai := rw.raft.AppliedIndex()
-			li := rw.raft.LastIndex()
-			logger.Debugf("current Raft index: %d/%d",
-				lai, li)
-			if lai == li {
-				return nil
-			}
-			time.Sleep(waitForUpdatesInterval)
-		}
-	}
-}
-
-func (rw *raftWrapper) WaitForPeer(ctx context.Context, pid string, depart bool) error {
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-			peers, err := rw.Peers(ctx)
-			if err != nil {
-				return err
-			}
-
-			if len(peers) == 1 && pid == peers[0] && depart {
-				return errWaitingForSelf
-			}
-
-			found := find(peers, pid)
-
-			// departing
-			if depart && !found {
-				return nil
-			}
-
-			// joining
-			if !depart && found {
-				return nil
-			}
-
-			time.Sleep(50 * time.Millisecond)
-		}
-	}
-}
-
-// Snapshot tells Raft to take a snapshot.
-func (rw *raftWrapper) Snapshot() error {
-	future := rw.raft.Snapshot()
-	err := future.Error()
-	if err != nil && err.Error() != hraft.ErrNothingNewToSnapshot.Error() {
-		return err
-	}
-	return nil
-}
-
-// snapshotOnShutdown attempts to take a snapshot before a shutdown.
-// Snapshotting might fail if the raft applied index is not the last index.
-// This waits for the updates and tries to take a snapshot when the
-// applied index is up to date.
-// It will retry if the snapshot still fails, in case more updates have arrived.
-// If waiting for updates times-out, it will not try anymore, since something
-// is wrong. This is a best-effort solution as there is no way to tell Raft
-// to stop processing entries because we want to take a snapshot before
-// shutting down.
-func (rw *raftWrapper) snapshotOnShutdown() error {
-	var err error
-	for i := 0; i < maxShutdownSnapshotRetries; i++ {
-		ctx, cancel := context.WithTimeout(context.Background(), waitForUpdatesShutdownTimeout)
-		err = rw.WaitForUpdates(ctx)
-		cancel()
-		if err != nil {
-			logger.Warn("timed out waiting for state updates before shutdown. Snapshotting may fail")
-			return rw.Snapshot()
-		}
-
-		err = rw.Snapshot()
-		if err == nil {
-			return nil // things worked
-		}
-
-		// There was an error
-		err = errors.New("could not snapshot raft: " + err.Error())
-		logger.Warnf("retrying to snapshot (%d/%d)...", i+1, maxShutdownSnapshotRetries)
-	}
-	return err
-}
-
-// Shutdown shutdown Raft and closes the BoltDB.
-func (rw *raftWrapper) Shutdown(ctx context.Context) error {
-
-	rw.cancel()
-
-	var finalErr error
-
-	err := rw.snapshotOnShutdown()
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	future := rw.raft.Shutdown()
-	err = future.Error()
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	err = rw.boltdb.Close() // important!
-	if err != nil {
-		finalErr = multierr.Append(finalErr, err)
-	}
-
-	return finalErr
-}
-
-// AddPeer adds a peer to Raft
-func (rw *raftWrapper) AddPeer(ctx context.Context, peerId peer.ID) error {
-
-	// Check that we don't have it to not waste
-	// log entries if so.
-	peers, err := rw.Peers(ctx)
-	if err != nil {
-		return err
-	}
-	if find(peers, peerId.String()) {
-		logger.Infof("%s is already a raft peerStr", peerId.String())
-		return nil
-	}
-
-	err = rw.host.Connect(ctx, peer.AddrInfo{ID: peerId})
-	if err != nil {
-		return err
-	}
-
-	future := rw.raft.AddVoter(
-		hraft.ServerID(peerId.String()),
-		hraft.ServerAddress(peerId.String()),
-		0,
-		0,
-	) // TODO: Extra cfg value?
-	err = future.Error()
-	if err != nil {
-		logger.Error("raft cannot add peer: ", err)
-	}
-	return err
-}
-
-// RemovePeer removes a peer from Raft
-func (rw *raftWrapper) RemovePeer(ctx context.Context, peer string) error {
-	// Check that we have it to not waste
-	// log entries if we don't.
-	peers, err := rw.Peers(ctx)
-	if err != nil {
-		return err
-	}
-	if !find(peers, peer) {
-		logger.Infof("%s is not among raft peers", peer)
-		return nil
-	}
-
-	if len(peers) == 1 && peers[0] == peer {
-		return errors.New("cannot remove ourselves from a 1-peer cluster")
-	}
-
-	rmFuture := rw.raft.RemoveServer(
-		hraft.ServerID(peer),
-		0,
-		0,
-	)
-	err = rmFuture.Error()
-	if err != nil {
-		logger.Error("raft cannot remove peer: ", err)
-		return err
-	}
-
-	return nil
-}
-
-// Leader returns Raft's leader. It may be an empty string if
-// there is no leader or it is unknown.
-func (rw *raftWrapper) Leader(ctx context.Context) string {
-	return string(rw.raft.Leader())
-}
-
-func (rw *raftWrapper) Peers(ctx context.Context) ([]string, error) {
-	ids := make([]string, 0)
-
-	configFuture := rw.raft.GetConfiguration()
-	if err := configFuture.Error(); err != nil {
-		return nil, err
-	}
-
-	for _, server := range configFuture.Configuration().Servers {
-		ids = append(ids, string(server.ID))
-	}
-
-	return ids, nil
-}
-
-// CleanupRaft moves the current data folder to a backup location
-//func CleanupRaft(cfg *Config) error {
-//	dataFolder := cfg.GetDataFolder()
-//	keep := cfg.BackupsRotate
-//
-//	meta, _, err := latestSnapshot(dataFolder)
-//	if meta == nil && err == nil {
-//		// no snapshots at all. Avoid creating backups
-//		// from empty state folders.
-//		logger.Infof("cleaning empty Raft data folder (%s)", dataFolder)
-//		os.RemoveAll(dataFolder)
-//		return nil
-//	}
-//
-//	logger.Infof("cleaning and backing up Raft data folder (%s)", dataFolder)
-//	dbh := newDataBackupHelper(dataFolder, keep)
-//	err = dbh.makeBackup()
-//	if err != nil {
-//		logger.Warn(err)
-//		logger.Warn("the state could not be cleaned properly")
-//		logger.Warn("manual intervention may be needed before starting cluster again")
-//	}
-//	return nil
-//}
-
-// only call when Raft is shutdown
-func (rw *raftWrapper) Clean() error {
-	//return CleanupRaft(rw.config)
-	return nil
-}
-
-func find(s []string, elem string) bool {
-	for _, selem := range s {
-		if selem == elem {
-			return true
-		}
-	}
-	return false
-}

lib/harmony/harmonydb/doc.go

@@ -0,0 +1,35 @@
+/*
+# HarmonyDB provides database abstractions over SP-wide Postgres-compatible instance(s).
+
+# Features
+
+	Rolling to secondary database servers on connection failure
+	Convenience features for Go + SQL
+	Prevention of SQL injection vulnerabilities
+	Monitors behavior via Prometheus stats and logging of errors.
+
+# Usage
+
+Processes should use New() to instantiate a *DB and keep it.
+Consumers can use this *DB concurrently.
+Creating and changing tables & views should happen in ./sql/ folder.
+Name the file "today's date" in the format: YYYYMMDD.sql (ex: 20231231.sql for the year's last day)
+
+	a. CREATE TABLE should NOT have a schema:
+		GOOD: CREATE TABLE foo ();
+		BAD:  CREATE TABLE me.foo ();
+	b. Schema is managed for you. It provides isolation for integraton tests & multi-use.
+	c. Git Merges: All run once, so old-after-new is OK when there are no deps.
+	d. NEVER change shipped sql files. Have later files make corrections.
+	e. Anything not ran will be ran, so an older date making it to master is OK.
+
+Write SQL with context, raw strings, and args:
+
+	name := "Alice"
+	var ID int
+	err := QueryRow(ctx, "SELECT id from people where first_name=?", name).Scan(&ID)
+	fmt.Println(ID)
+
+Note: Scan() is column-oriented, while Select() & StructScan() is field name/tag oriented.
+*/
+package harmonydb

lib/harmony/harmonydb/harmonydb.go

@@ -0,0 +1,301 @@
+package harmonydb
+
+import (
+	"context"
+	"embed"
+	"fmt"
+	"math/rand"
+	"net"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/node/config"
+)
+
+type ITestID string
+
+// ItestNewID see ITestWithID doc
+func ITestNewID() ITestID {
+	return ITestID(strconv.Itoa(rand.Intn(99999)))
+}
+
+type DB struct {
+	pgx       *pgxpool.Pool
+	cfg       *pgxpool.Config
+	schema    string
+	hostnames []string
+	BTFPOnce  sync.Once
+	BTFP      atomic.Uintptr
+}
+
+var logger = logging.Logger("harmonydb")
+
+// NewFromConfig is a convenience function.
+// In usage:
+//
+//	db, err := NewFromConfig(config.HarmonyDB)  // in binary init
+func NewFromConfig(cfg config.HarmonyDB) (*DB, error) {
+	return New(
+		cfg.Hosts,
+		cfg.Username,
+		cfg.Password,
+		cfg.Database,
+		cfg.Port,
+		"",
+	)
+}
+
+func NewFromConfigWithITestID(cfg config.HarmonyDB) func(id ITestID) (*DB, error) {
+	return func(id ITestID) (*DB, error) {
+		return New(
+			cfg.Hosts,
+			cfg.Username,
+			cfg.Password,
+			cfg.Database,
+			cfg.Port,
+			id,
+		)
+	}
+}
+
+// New is to be called once per binary to establish the pool.
+// log() is for errors. It returns an upgraded database's connection.
+// This entry point serves both production and integration tests, so it's more DI.
+func New(hosts []string, username, password, database, port string, itestID ITestID) (*DB, error) {
+	itest := string(itestID)
+	connString := ""
+	if len(hosts) > 0 {
+		connString = "host=" + hosts[0] + " "
+	}
+	for k, v := range map[string]string{"user": username, "password": password, "dbname": database, "port": port} {
+		if strings.TrimSpace(v) != "" {
+			connString += k + "=" + v + " "
+		}
+	}
+
+	schema := "lotus"
+	if itest != "" {
+		schema = "itest_" + itest
+	}
+
+	if err := ensureSchemaExists(connString, schema); err != nil {
+		return nil, err
+	}
+	cfg, err := pgxpool.ParseConfig(connString + "search_path=" + schema)
+	if err != nil {
+		return nil, err
+	}
+
+	// enable multiple fallback hosts.
+	for _, h := range hosts[1:] {
+		cfg.ConnConfig.Fallbacks = append(cfg.ConnConfig.Fallbacks, &pgconn.FallbackConfig{Host: h})
+	}
+
+	cfg.ConnConfig.OnNotice = func(conn *pgconn.PgConn, n *pgconn.Notice) {
+		logger.Debug("database notice: " + n.Message + ": " + n.Detail)
+		DBMeasures.Errors.M(1)
+	}
+
+	db := DB{cfg: cfg, schema: schema, hostnames: hosts} // pgx populated in AddStatsAndConnect
+	if err := db.addStatsAndConnect(); err != nil {
+		return nil, err
+	}
+
+	return &db, db.upgrade()
+}
+
+type tracer struct {
+}
+
+type ctxkey string
+
+const SQL_START = ctxkey("sqlStart")
+const SQL_STRING = ctxkey("sqlString")
+
+func (t tracer) TraceQueryStart(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
+	return context.WithValue(context.WithValue(ctx, SQL_START, time.Now()), SQL_STRING, data.SQL)
+}
+func (t tracer) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryEndData) {
+	DBMeasures.Hits.M(1)
+	ms := time.Since(ctx.Value(SQL_START).(time.Time)).Milliseconds()
+	DBMeasures.TotalWait.M(ms)
+	DBMeasures.Waits.Observe(float64(ms))
+	if data.Err != nil {
+		DBMeasures.Errors.M(1)
+	}
+	logger.Debugw("SQL run",
+		"query", ctx.Value(SQL_STRING).(string),
+		"err", data.Err,
+		"rowCt", data.CommandTag.RowsAffected(),
+		"milliseconds", ms)
+}
+
+func (db *DB) GetRoutableIP() (string, error) {
+	tx, err := db.pgx.Begin(context.Background())
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = tx.Rollback(context.Background()) }()
+	local := tx.Conn().PgConn().Conn().LocalAddr()
+	addr, ok := local.(*net.TCPAddr)
+	if !ok {
+		return "", fmt.Errorf("could not get local addr from %v", addr)
+	}
+	return addr.IP.String(), nil
+}
+
+// addStatsAndConnect connects a prometheus logger. Be sure to run this before using the DB.
+func (db *DB) addStatsAndConnect() error {
+
+	db.cfg.ConnConfig.Tracer = tracer{}
+
+	hostnameToIndex := map[string]float64{}
+	for i, h := range db.hostnames {
+		hostnameToIndex[h] = float64(i)
+	}
+	db.cfg.AfterConnect = func(ctx context.Context, c *pgx.Conn) error {
+		s := db.pgx.Stat()
+		DBMeasures.OpenConnections.M(int64(s.TotalConns()))
+		DBMeasures.WhichHost.Observe(hostnameToIndex[c.Config().Host])
+
+		//FUTURE place for any connection seasoning
+		return nil
+	}
+
+	// Timeout the first connection so we know if the DB is down.
+	ctx, ctxClose := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
+	defer ctxClose()
+	var err error
+	db.pgx, err = pgxpool.NewWithConfig(ctx, db.cfg)
+	if err != nil {
+		logger.Error(fmt.Sprintf("Unable to connect to database: %v\n", err))
+		return err
+	}
+	return nil
+}
+
+// ITestDeleteAll will delete everything created for "this" integration test.
+// This must be called at the end of each integration test.
+func (db *DB) ITestDeleteAll() {
+	if !strings.HasPrefix(db.schema, "itest_") {
+		fmt.Println("Warning: this should never be called on anything but an itest schema.")
+		return
+	}
+	defer db.pgx.Close()
+	_, err := db.pgx.Exec(context.Background(), "DROP SCHEMA "+db.schema+" CASCADE")
+	if err != nil {
+		fmt.Println("warning: unclean itest shutdown: cannot delete schema: " + err.Error())
+		return
+	}
+}
+
+var schemaREString = "^[A-Za-z0-9_]+$"
+var schemaRE = regexp.MustCompile(schemaREString)
+
+func ensureSchemaExists(connString, schema string) error {
+	// FUTURE allow using fallback DBs for start-up.
+	ctx, cncl := context.WithDeadline(context.Background(), time.Now().Add(3*time.Second))
+	p, err := pgx.Connect(ctx, connString)
+	defer cncl()
+	if err != nil {
+		return xerrors.Errorf("unable to connect to db: %s, err: %v", connString, err)
+	}
+	defer func() { _ = p.Close(context.Background()) }()
+
+	if len(schema) < 5 || !schemaRE.MatchString(schema) {
+		return xerrors.New("schema must be of the form " + schemaREString + "\n Got: " + schema)
+	}
+	_, err = p.Exec(context.Background(), "CREATE SCHEMA IF NOT EXISTS "+schema)
+	if err != nil {
+		return xerrors.Errorf("cannot create schema: %w", err)
+	}
+	return nil
+}
+
+//go:embed sql
+var fs embed.FS
+
+func (db *DB) upgrade() error {
+	// Does the version table exist? if not, make it.
+	// NOTE: This cannot change except via the next sql file.
+	_, err := db.Exec(context.Background(), `CREATE TABLE IF NOT EXISTS base (
+		id SERIAL PRIMARY KEY,
+		entry CHAR(12),
+		applied TIMESTAMP DEFAULT current_timestamp
+	)`)
+	if err != nil {
+		logger.Error("Upgrade failed.")
+		return xerrors.Errorf("Cannot create base table %w", err)
+	}
+
+	// __Run scripts in order.__
+
+	landed := map[string]bool{}
+	{
+		var landedEntries []struct{ Entry string }
+		err = db.Select(context.Background(), &landedEntries, "SELECT entry FROM base")
+		if err != nil {
+			logger.Error("Cannot read entries: " + err.Error())
+			return xerrors.Errorf("cannot read entries: %w", err)
+		}
+		for _, l := range landedEntries {
+			landed[l.Entry[:8]] = true
+		}
+	}
+	dir, err := fs.ReadDir("sql")
+	if err != nil {
+		logger.Error("Cannot read fs entries: " + err.Error())
+		return err
+	}
+	sort.Slice(dir, func(i, j int) bool { return dir[i].Name() < dir[j].Name() })
+
+	if len(dir) == 0 {
+		logger.Error("No sql files found.")
+	}
+	for _, e := range dir {
+		name := e.Name()
+		if !strings.HasSuffix(name, ".sql") {
+			logger.Debug("Must have only SQL files here, found: " + name)
+			continue
+		}
+		if landed[name[:8]] {
+			logger.Debug("DB Schema " + name + " already applied.")
+			continue
+		}
+		file, err := fs.ReadFile("sql/" + name)
+		if err != nil {
+			logger.Error("weird embed file read err")
+			return err
+		}
+		for _, s := range strings.Split(string(file), ";") { // Implement the changes.
+			if len(strings.TrimSpace(s)) == 0 {
+				continue
+			}
+			_, err = db.pgx.Exec(context.Background(), s)
+			if err != nil {
+				msg := fmt.Sprintf("Could not upgrade! File %s, Query: %s, Returned: %s", name, s, err.Error())
+				logger.Error(msg)
+				return xerrors.New(msg) // makes devs lives easier by placing message at the end.
+			}
+		}
+
+		// Mark Completed.
+		_, err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name[:8])
+		if err != nil {
+			logger.Error("Cannot update base: " + err.Error())
+			return xerrors.Errorf("cannot insert into base: %w", err)
+		}
+	}
+	return nil
+}

lib/harmony/harmonydb/metrics.go

@@ -0,0 +1,77 @@
+package harmonydb
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+
+	"github.com/filecoin-project/lotus/metrics"
+)
+
+var (
+	dbTag, _         = tag.NewKey("db_name")
+	pre              = "harmonydb_base_"
+	waitsBuckets     = []float64{0, 10, 20, 30, 50, 80, 130, 210, 340, 550, 890}
+	whichHostBuckets = []float64{0, 1, 2, 3, 4, 5}
+)
+
+// DBMeasures groups all db metrics.
+var DBMeasures = struct {
+	Hits            *stats.Int64Measure
+	TotalWait       *stats.Int64Measure
+	Waits           prometheus.Histogram
+	OpenConnections *stats.Int64Measure
+	Errors          *stats.Int64Measure
+	WhichHost       prometheus.Histogram
+}{
+	Hits:      stats.Int64(pre+"hits", "Total number of uses.", stats.UnitDimensionless),
+	TotalWait: stats.Int64(pre+"total_wait", "Total delay. A numerator over hits to get average wait.", stats.UnitMilliseconds),
+	Waits: prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    pre + "waits",
+		Buckets: waitsBuckets,
+		Help:    "The histogram of waits for query completions.",
+	}),
+	OpenConnections: stats.Int64(pre+"open_connections", "Total connection count.", stats.UnitDimensionless),
+	Errors:          stats.Int64(pre+"errors", "Total error count.", stats.UnitDimensionless),
+	WhichHost: prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    pre + "which_host",
+		Buckets: whichHostBuckets,
+		Help:    "The index of the hostname being used",
+	}),
+}
+
+// CacheViews groups all cache-related default views.
+func init() {
+	metrics.RegisterViews(
+		&view.View{
+			Measure:     DBMeasures.Hits,
+			Aggregation: view.Sum(),
+			TagKeys:     []tag.Key{dbTag},
+		},
+		&view.View{
+			Measure:     DBMeasures.TotalWait,
+			Aggregation: view.Sum(),
+			TagKeys:     []tag.Key{dbTag},
+		},
+		&view.View{
+			Measure:     DBMeasures.OpenConnections,
+			Aggregation: view.LastValue(),
+			TagKeys:     []tag.Key{dbTag},
+		},
+		&view.View{
+			Measure:     DBMeasures.Errors,
+			Aggregation: view.Sum(),
+			TagKeys:     []tag.Key{dbTag},
+		},
+	)
+	err := prometheus.Register(DBMeasures.Waits)
+	if err != nil {
+		panic(err)
+	}
+
+	err = prometheus.Register(DBMeasures.WhichHost)
+	if err != nil {
+		panic(err)
+	}
+}

lib/harmony/harmonydb/sql/20230706.sql

@@ -0,0 +1,7 @@
+CREATE TABLE itest_scratch (
+    id SERIAL PRIMARY KEY,
+	content TEXT,
+    some_int INTEGER,
+    second_int INTEGER,
+	update_time TIMESTAMP DEFAULT current_timestamp
+)

lib/harmony/harmonydb/sql/20230712.sql

@@ -0,0 +1,45 @@
+create table sector_location
+(
+    miner_id         bigint    not null,
+    sector_num       bigint    not null,
+    sector_filetype  int   not null,
+    storage_id       varchar not null,
+    is_primary       bool,
+    read_ts          timestamp(6),
+    read_refs        int,
+    write_ts         timestamp(6),
+    write_lock_owner varchar,
+    constraint sectorlocation_pk
+        primary key (miner_id, sector_num, sector_filetype, storage_id)
+);
+
+alter table sector_location
+    alter column read_refs set not null;
+
+alter table sector_location
+    alter column read_refs set default 0;
+
+create table storage_path
+(
+    "storage_id"  varchar not null
+        constraint "storage_path_pkey"
+            primary key,
+    "urls"       varchar, -- comma separated list of urls
+    "weight"     bigint,
+    "max_storage" bigint,
+    "can_seal"    bool,
+    "can_store"   bool,
+    "groups"     varchar, -- comma separated list of group names
+    "allow_to"    varchar, -- comma separated list of allowed groups
+    "allow_types" varchar,  -- comma separated list of allowed file types
+    "deny_types"  varchar, -- comma separated list of denied file types
+
+    "capacity" bigint,
+    "available" bigint,
+    "fs_available" bigint,
+    "reserved" bigint,
+    "used" bigint,
+    "last_heartbeat" timestamp(6),
+    "heartbeat_err"  varchar
+);
+

lib/harmony/harmonydb/sql/20230719.sql

@@ -0,0 +1,51 @@
+/* For HarmonyTask base implementation. */
+
+CREATE TABLE harmony_machines (
+    id SERIAL PRIMARY KEY NOT NULL,
+    last_contact TIMESTAMP NOT NULL DEFAULT current_timestamp,
+    host_and_port varchar(300) NOT NULL, 
+    cpu INTEGER NOT NULL, 
+    ram BIGINT NOT NULL, 
+    gpu FLOAT NOT NULL
+);
+
+CREATE TABLE harmony_task (
+    id SERIAL PRIMARY KEY NOT NULL,
+    initiated_by INTEGER,     
+    update_time TIMESTAMP NOT NULL DEFAULT current_timestamp,
+    posted_time TIMESTAMP NOT NULL,
+    owner_id INTEGER REFERENCES harmony_machines (id) ON DELETE SET NULL, 
+    added_by INTEGER NOT NULL,
+    previous_task INTEGER,
+    name varchar(16) NOT NULL
+);
+COMMENT ON COLUMN harmony_task.initiated_by IS 'The task ID whose completion occasioned this task.';
+COMMENT ON COLUMN harmony_task.owner_id IS 'The foreign key to harmony_machines.';
+COMMENT ON COLUMN harmony_task.name IS 'The name of the task type.';
+COMMENT ON COLUMN harmony_task.owner_id IS 'may be null if between owners or not yet taken';
+COMMENT ON COLUMN harmony_task.update_time IS 'When it was last modified. not a heartbeat';
+
+CREATE TABLE harmony_task_history (
+    id SERIAL PRIMARY KEY NOT NULL,  
+    task_id INTEGER NOT NULL, 
+    name VARCHAR(16) NOT NULL,
+    posted TIMESTAMP NOT NULL, 
+    work_start TIMESTAMP NOT NULL, 
+    work_end TIMESTAMP NOT NULL, 
+    result BOOLEAN NOT NULL, 
+    err varchar
+);
+COMMENT ON COLUMN harmony_task_history.result IS 'Use to detemine if this was a successful run.';
+
+CREATE TABLE harmony_task_follow (
+    id SERIAL PRIMARY KEY NOT NULL,  
+    owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
+    to_type VARCHAR(16) NOT NULL,
+    from_type VARCHAR(16) NOT NULL
+);
+
+CREATE TABLE harmony_task_impl (
+    id SERIAL PRIMARY KEY NOT NULL,  
+    owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
+    name VARCHAR(16) NOT NULL
+);

lib/harmony/harmonydb/sql/20230823.sql

@@ -0,0 +1,48 @@
+create table wdpost_partition_tasks
+(
+    task_id              bigint not null
+        constraint wdpost_partition_tasks_pk
+            primary key,
+    sp_id                bigint not null,
+    proving_period_start bigint not null,
+    deadline_index       bigint not null,
+    partition_index      bigint not null,
+    constraint wdpost_partition_tasks_identity_key
+        unique (sp_id, proving_period_start, deadline_index, partition_index)
+);
+
+comment on column wdpost_partition_tasks.task_id is 'harmonytask task ID';
+comment on column wdpost_partition_tasks.sp_id is 'storage provider ID';
+comment on column wdpost_partition_tasks.proving_period_start is 'proving period start';
+comment on column wdpost_partition_tasks.deadline_index is 'deadline index within the proving period';
+comment on column wdpost_partition_tasks.partition_index is 'partition index within the deadline';
+
+create table wdpost_proofs
+(
+    sp_id                bigint not null,
+    proving_period_start bigint not null,
+    deadline             bigint not null,
+    partition            bigint not null,
+    submit_at_epoch      bigint not null,
+    submit_by_epoch      bigint not null,
+    proof_params         bytea,
+
+    submit_task_id       bigint,
+    message_cid          text,
+
+    constraint wdpost_proofs_identity_key
+        unique (sp_id, proving_period_start, deadline, partition)
+);
+
+create table wdpost_recovery_tasks
+(
+    task_id              bigint not null
+        constraint wdpost_recovery_tasks_pk
+            primary key,
+    sp_id                bigint not null,
+    proving_period_start bigint not null,
+    deadline_index       bigint not null,
+    partition_index      bigint not null,
+    constraint wdpost_recovery_tasks_identity_key
+        unique (sp_id, proving_period_start, deadline_index, partition_index)
+);