Update one test to pass optional block param to EthEstimateGas

* exclude reverted events from results returned by eth_getLogs

* unit test

* update CHANGELOG.md

.circleci/config.yml

@@ -7,12 +7,12 @@ executors:
   golang:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.19.12
+      - image: cimg/go:1.20.7
     resource_class: medium+
   golang-2xl:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.19.12
+      - image: cimg/go:1.20.7
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -997,6 +997,7 @@ workflows:
           suite: utest-unit-cli
           target: "./cli/... ./cmd/... ./api/..."
           get-params: true
+          executor: golang-2xl
       - test:
           name: test-unit-node
           requires:
@@ -1004,12 +1005,14 @@ workflows:
           suite: utest-unit-node
           target: "./node/..."
           
+          
       - test:
           name: test-unit-rest
           requires:
             - build
           suite: utest-unit-rest
-          target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
+          target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
+          
           executor: golang-2xl
       - test:
           name: test-unit-storage
@@ -1018,6 +1021,7 @@ workflows:
           suite: utest-unit-storage
           target: "./storage/... ./extern/..."
           
+          
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           requires:

.circleci/gen.go

@@ -65,6 +65,8 @@ func main() {
 			if err != nil {
 				panic(err)
 			}
+			// Redundantly flag both absolute and relative paths as excluded
+			excluded[filepath.Join(repo, s)] = struct{}{}
 			excluded[e] = struct{}{}
 		}
 	}

.circleci/template.yml

@@ -7,12 +7,12 @@ executors:
   golang:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.19.12
+      - image: cimg/go:1.20.7
     resource_class: medium+
   golang-2xl:
     docker:
       # Must match GO_VERSION_MIN in project root
-      - image: cimg/go:1.19.12
+      - image: cimg/go:1.20.7
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -558,6 +558,7 @@ workflows:
           suite: utest-[[ $suite ]]
           target: "[[ $pkgs ]]"
           [[if eq $suite "unit-cli"]]get-params: true[[end]]
+          [[if eq $suite "unit-cli"]]executor: golang-2xl[[end]]
           [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]]
       [[- end]]
       - test:

.github/ISSUE_TEMPLATE/task.md

@@ -0,0 +1,31 @@
+---
+name: New Task
+about: A larger yet well-scoped task
+title: '<title>'
+labels: Needs Triage
+assignees: ''
+
+---
+
+## User Story
+<!-- Why? -->
+
+## Acceptance Criteria
+<!-- What? -->
+<!-- add description-->
+
+```[tasklist]
+### Deliverables
+
+```
+
+## Technical Breakdown
+```[tasklist]
+### Development
+
+```
+
+```[tasklist]
+### Testing
+
+```

.github/pull_request_template.md

@@ -16,6 +16,7 @@ Before you mark the PR ready for review, please make sure that:
   - example: ` fix: mempool: Introduce a cache for valid signatures`
   - `PR type`: fix, feat, build, chore, ci, docs, perf, refactor, revert, style, test
   - `area`, e.g. api, chain, state, market, mempool, multisig, networking, paych, proving, sealing, wallet, deps
+- [ ] If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section.
 - [ ] New features have usage guidelines and / or documentation updates in
   - [ ] [Lotus Documentation](https://lotus.filecoin.io)
   - [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)

CHANGELOG.md

@@ -1,9 +1,241 @@
 # Lotus changelog
 
 # UNRELEASED
+- chore: Auto remove local chain data when importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
+- feat: metric: export Mpool message count ([filecoin-project/lotus#11361](https://github.com/filecoin-project/lotus/pull/11361))
+- feat: sealing: load SectorsSummary from sealing SectorStats instead of calling API each time ([filecoin-project/lotus#11353](https://github.com/filecoin-project/lotus/pull/11353))
+- fix: miner info: Show correct sector state counts ([filecoin-project/lotus#11456](https://github.com/filecoin-project/lotus/pull/11456))
+- feat: add support for specifying block number when calling `eth_estimateGas` ([filecoin-project/lotus#11462](https://github.com/filecoin-project/lotus/pull/11462)).
+
+## Improvements
+- fix: exchange: allow up to 10k messages per block ([filecoin-project/lotus#11506](https://github.com/filecoin-project/lotus/pull/11506))
+
+# v1.25.1 / 2023-12-09
+
+This is a **highly recommended PATCH RELEASE.**  The patch release fixes the issue were node operators trying to catch up sync were unable to sync large message blocks/epochs due to an increased number of messages on the network.
+
+This patch release allows for up to 10k messages per block. Additionally, it introduces a limit on the amount of data that can be read at once, ensuring the system can handle worst-case scenarios.
+
+# v 1.25.0 / 2023-11-22
+
+This is a highly recommended feature release of Lotus. This optional release supports the Filecoin network version 21 upgrade, codenamed Watermelon 🍉, in addition to the numerous improvements and enhancements for node operators, ETH RPC-providers and storage providers.
+
+**The Filecoin network upgrade v21, codenamed Watermelon 🍉, is at epoch 3469380 - 2023-12-12T13:30:00Z**
+
+The full list of [protocol improvements delivered in the network upgrade can be found here](https://github.com/filecoin-project/core-devs/blob/master/Network%20Upgrades/v21.md).
+
+## ☢️ Upgrade Warnings ☢️
+
+- Read through the [changelog of the mandatory v1.24.0 release](https://github.com/filecoin-project/lotus/releases/tag/v1.24.0). Especially the `Migration` and `v12 Builtin Actor Bundle` sections.
+- Please remove and clone a new Lotus repo (`git clone https://github.com/filecoin-project/lotus.git`) when upgrading to this release. 
+- This feature release requires a minimum Go version of v1.20.7 or higher to successfully build Lotus. Go version 1.21.x is not supported yet.
+- EthRPC providers, please check out the [new tracing API to Lotus RPC](https://github.com/filecoin-project/lotus/pull/11100)
+
+## ⭐️ Highlights ⭐️
+
+**Unsealing bugfixes and enhancements**
+
+This feature release introduces significant improvements and bugfixes with regards to unsealing, and ensures that unsealing operates as one would expect. Consequently, unsealing of all sector types (deal sectors, snap-sectors without sector keys, and snap-sectors with sector keys) now all function seamlessly.
+
+Some additional unsealing improvements are:
+- Unsealing on workers with only sealing paths works. :tada:
+- Transferring unsealed files to long-term storage upon successful unsealing. :arrow_right:
+- Ensuring no residual files in sealing paths post a successful unsealing operation. :broom:
+
+**SupraSeal C2**
+
+Lotus-workers can now be built to leverage the SupraSeal C2 sealing optimizations in your sealing pipeline. The code optimizations are currently behind the `FFI_USE_CUDA_SUPRASEAL` feature flag. We advice users to test this feature on a test-network, before trying to use it on the mainnet. Users can test out the feature by building their lotus-workers by exporting the `FFI_USE_CUDA_SUPRASEAL=1` enviroment variable, and building from source. For questions about the SupraSeal C2 sealing optimizations, reach out in the #fil-proofs or the #dsa-sealing slack channel.
 
 ## New features
-- feat: Add move-partition command ([filecoin-project/lotus#11290](https://github.com/filecoin-project/lotus/pull/11290))
+- feat: add Eip155ChainID to StateGetNetworkParams ([filecoin-project/lotus#10987](https://github.com/filecoin-project/lotus/pull/10987))
+- feat: profiling: state summary and visualization ([filecoin-project/lotus#11012](https://github.com/filecoin-project/lotus/pull/11012))
+- feat: snapshot: remove existing chain ([filecoin-project/lotus#11032](https://github.com/filecoin-project/lotus/pull/11032))
+- feat: Add a metric to display pruning of the node's peer ([filecoin-project/lotus#11058](https://github.com/filecoin-project/lotus/pull/11058))
+- feat:shed:gather partition metadata ([filecoin-project/lotus#11078](https://github.com/filecoin-project/lotus/pull/11078))
+- feat: vm: allow raw "cbor" in state and use the new go-multicodec ([filecoin-project/lotus#11081](https://github.com/filecoin-project/lotus/pull/11081))
+- Add new lotus-shed command for backfillling actor events ([filecoin-project/lotus#11088](https://github.com/filecoin-project/lotus/pull/11088))
+- feat: Add new tracing API ([filecoin-project/lotus#11100](https://github.com/filecoin-project/lotus/pull/11100))
+- feat: FVM: do not error on unsuccessful implicit messages ([filecoin-project/lotus#11120](https://github.com/filecoin-project/lotus/pull/11120))
+- feat: chain node: Move consensus slasher to internal service ([filecoin-project/lotus#11126](https://github.com/filecoin-project/lotus/pull/11126))
+- feat: miner: implement FRC-0051 ([filecoin-project/lotus#11157](https://github.com/filecoin-project/lotus/pull/11157))
+  - feat: chainstore: FRC-0051: Remove all equivocated blocks from tipsets ([filecoin-project/lotus#11104](https://github.com/filecoin-project/lotus/pull/11104))
+  - feat: miner: 2 minor refactors ([filecoin-project/lotus#11158](https://github.com/filecoin-project/lotus/pull/11158))
+- feat: refactor: return randomness base to FVM without hashing ([filecoin-project/lotus#11167](https://github.com/filecoin-project/lotus/pull/11167))
+- feat: Lotus Gateway: add allocation and claim related GET APIs to gateway ([filecoin-project/lotus#11183](https://github.com/filecoin-project/lotus/pull/11183))
+- feat: shed: Add exec traces to `lotus-shed msg` ([filecoin-project/lotus#11188](https://github.com/filecoin-project/lotus/pull/11188))
+- feat: miner: defensive check for equivocation ([filecoin-project/lotus#11328](https://github.com/filecoin-project/lotus/pull/11328))
+
+## Improvements
+- feat: daemon: improvemens to the consensus slasher ([filecoin-project/lotus#10979](https://github.com/filecoin-project/lotus/pull/10979))
+- fix: Snapdeals unsealing fixes ([filecoin-project/lotus#11011](https://github.com/filecoin-project/lotus/pull/11011))
+- refactor: Make all validation error actions explicit ([filecoin-project/lotus#11016](https://github.com/filecoin-project/lotus/pull/11016))
+- feat: shed: command for decoding block headers ([filecoin-project/lotus#11031](https://github.com/filecoin-project/lotus/pull/11031))
+- fix: stores: Tune down StorageDeclareSector` log-lvl ([filecoin-project/lotus#11045](https://github.com/filecoin-project/lotus/pull/11045))
+- feat: types: apply a max length when decoding events ([filecoin-project/lotus#11054](https://github.com/filecoin-project/lotus/pull/11054))
+- feat: slasher: improve UX ([filecoin-project/lotus#11060](https://github.com/filecoin-project/lotus/pull/11060))
+- feat: daemon: improvemens to the consensus slasher ([filecoin-project/lotus#11063](https://github.com/filecoin-project/lotus/pull/11063))
+- fix: events: Improve performance of event migration from V1 to V2 ([filecoin-project/lotus#11064](https://github.com/filecoin-project/lotus/pull/11064))
+- feat:lotus-bench:AMT benchmarking ([filecoin-project/lotus#11075](https://github.com/filecoin-project/lotus/pull/11075))
+- fix: DecodeRLP can panic ([filecoin-project/lotus#11079](https://github.com/filecoin-project/lotus/pull/11079))
+- fix: daemon: set real beacon schedule when importing chain ([filecoin-project/lotus#11080](https://github.com/filecoin-project/lotus/pull/11080))
+- fix: ethtypes: handle length overflow case ([filecoin-project/lotus#11082](https://github.com/filecoin-project/lotus/pull/11082))
+- chore: stmgr: migrations: do not log noisily on cache misses ([filecoin-project/lotus#11083](https://github.com/filecoin-project/lotus/pull/11083))
+- feat: daemon: import: only setup stmgr if validating chain (#11084) ([filecoin-project/lotus#11084](https://github.com/filecoin-project/lotus/pull/11084))
+- fix: sealing pipeline: Fix PC1 retry loop ([filecoin-project/lotus#11087](https://github.com/filecoin-project/lotus/pull/11087))
+- chore: legacy syscalls: Cleanup ComputeUnsealedSectorCID ([filecoin-project/lotus#11119](https://github.com/filecoin-project/lotus/pull/11119))
+- sector import: fix evaluating randomness when importing a sector ([filecoin-project/lotus#11123](https://github.com/filecoin-project/lotus/pull/11123))
+- fix: cli: Only display `warning` if behind sync ([filecoin-project/lotus#11140](https://github.com/filecoin-project/lotus/pull/11140))
+- fix: worker: Support IPv6 formatted API-keys ([filecoin-project/lotus#11141](https://github.com/filecoin-project/lotus/pull/11141))
+- fix: sealing: Switch to calling PreCommitSectorBatch2 ([filecoin-project/lotus#11142](https://github.com/filecoin-project/lotus/pull/11142))
+- fix: downgrade harmless warning to debug ([filecoin-project/lotus#11145](https://github.com/filecoin-project/lotus/pull/11145))
+- fix: sealing: Fix RetryCommitWait loop when sector cron activation fails ([filecoin-project/lotus#11046](https://github.com/filecoin-project/lotus/pull/11046))
+- fix: gateway: return an error when an Eth filter is not found ([filecoin-project/lotus#11152](https://github.com/filecoin-project/lotus/pull/11152))
+- fix: chainstore: do not get stuck in unhappy equivocation cases ([filecoin-project/lotus#11159](https://github.com/filecoin-project/lotus/pull/11159))
+- fix: sealing: Run unsealing in the background for better ux ([filecoin-project/lotus#11177](https://github.com/filecoin-project/lotus/pull/11177))
+- fix: build: Allow lotus-wallet to be built independently ([filecoin-project/lotus#11187](https://github.com/filecoin-project/lotus/pull/11187))
+- fix: wallet: Make import handle SIGINT/SIGTERM ([filecoin-project/lotus#11190](https://github.com/filecoin-project/lotus/pull/11190))
+- fix: markets/dagstore: remove trace goroutine for dagstore wrapper ([filecoin-project/lotus#11191](https://github.com/filecoin-project/lotus/pull/11191))
+- fix: chain: Do not update message info cache until after message validation ([filecoin-project/lotus#11202](https://github.com/filecoin-project/lotus/pull/11202))
+- fix: chain: cancel long operations upon ctx cancelation ([filecoin-project/lotus#11206](https://github.com/filecoin-project/lotus/pull/11206))
+- fix(client): single-root error message ([filecoin-project/lotus#11214](https://github.com/filecoin-project/lotus/pull/11214))
+- fix: worker: Convert `DC_[SectorSize]_[ResourceRestriction]` if set ([filecoin-project/lotus#11224](https://github.com/filecoin-project/lotus/pull/11224))
+- chore: backport #11338 onto release/v1.25.0 ([filecoin-project/lotus#11350](https://github.com/filecoin-project/lotus/pull/11350))
+- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
+
+## Dependencies
+- deps: update go-libp2p to v0.28.1 ([filecoin-project/lotus#10998](https://github.com/filecoin-project/lotus/pull/10998))
+- deps: update go-libp2p to v0.29.2 ([filecoin-project/lotus#11164](https://github.com/filecoin-project/lotus/pull/11164))
+- deps: update go-libp2p to v0.30.0 ([filecoin-project/lotus#11189](https://github.com/filecoin-project/lotus/pull/11189))
+- fix: build: use tagged releases ([filecoin-project/lotus#11194](https://github.com/filecoin-project/lotus/pull/11194))
+- chore: test-vectors: update ([filecoin-project/lotus#11196](https://github.com/filecoin-project/lotus/pull/11196))
+- chore: backport #11365 to release/v1.25.0 ([filecoin-project/lotus#11369](https://github.com/filecoin-project/lotus/pull/11369))
+- chore: deps: update to go-state-types v0.12.8 ([filecoin-project/lotus#11339](https://github.com/filecoin-project/lotus/pull/11437))
+- chore: deps: update to final actors ([filecoin-project/lotus#11330](https://github.com/filecoin-project/lotus/pull/11440))
+- github.com/filecoin-project/go-amt-ipld/v4 (v4.0.0 -> v4.2.0)
+- github.com/filecoin-project/test-vectors/schema (v0.0.5 -> v0.0.7)
+
+## Others
+- chore: Extract stable release and post release portion outside of RC testing in template ([filecoin-project/lotus#11000](https://github.com/filecoin-project/lotus/pull/11000))
+- fix: docs: include FFI steps in lotus RELEASE_ISSUE_TEMPLATE ([filecoin-project/lotus#11047](https://github.com/filecoin-project/lotus/pull/11047))
+- chore: build: update to v1.23.4-dev ([filecoin-project/lotus#11049](https://github.com/filecoin-project/lotus/pull/11049))
+- fix: deflake: Use MockProofs ([filecoin-project/lotus#11059](https://github.com/filecoin-project/lotus/pull/11059))
+- fix: failing test: Tweak TestWindowPostV1P1NV20 test condition ([filecoin-project/lotus#11121](https://github.com/filecoin-project/lotus/pull/11121))
+- fix: CI: make test-unit-rest actually be the rest of the tests ([filecoin-project/lotus#11147](https://github.com/filecoin-project/lotus/pull/11147))
+- chore: merge releases into master ([filecoin-project/lotus#11154](https://github.com/filecoin-project/lotus/pull/11154))
+- tests: deflake: TestGetBlockByNumber ([filecoin-project/lotus#11155](https://github.com/filecoin-project/lotus/pull/11155))
+- tests: mac seal test ([filecoin-project/lotus#11180](https://github.com/filecoin-project/lotus/pull/11180))
+- tests: Take Download out of Sealer time ([filecoin-project/lotus#11182](https://github.com/filecoin-project/lotus/pull/11182))
+- feat: test: Test that verified clients can directly transfer datacap, creating allocations ([filecoin-project/lotus#11169](https://github.com/filecoin-project/lotus/pull/11169))
+- chore: merge feat/nv21 into master ([filecoin-project/lotus#11201](https://github.com/filecoin-project/lotus/pull/11201))
+- ci: Use larger executor for cli tests ([filecoin-project/lotus#11212](https://github.com/filecoin-project/lotus/pull/11212))
+- fix: dockerfile: Bump to Go 1.20.7 image ([filecoin-project/lotus#11221](https://github.com/filecoin-project/lotus/pull/11221))
+- docs: Update PR template to callout remembering to update CHANGELOG ([filecoin-project/lotus#11232](https://github.com/filecoin-project/lotus/pull/11232))
+- chore: release: 1.23.4rc1 prep ([filecoin-project/lotus#11248](https://github.com/filecoin-project/lotus/pull/11248))
+- chore: backport #11262 (#11265) ([filecoin-project/lotus#11265](https://github.com/filecoin-project/lotus/pull/11265))
+- chore: backport #11294 into `release/v1.23.4` ([filecoin-project/lotus#11295](https://github.com/filecoin-project/lotus/pull/11295))
+- chore: release: V1.25 rebase ([filecoin-project/lotus#11342](https://github.com/filecoin-project/lotus/pull/11342))
+- backport: tests: add SynthPorep layers to cachefiles ([filecoin-project/lotus#11344](https://github.com/filecoin-project/lotus/pull/11344))
+- chore: backport #11408 to release/v1.25.0 ([filecoin-project/lotus#11414](https://github.com/filecoin-project/lotus/pull/11414))
+- chore: backport calibnet lightweight patch ([filecoin-project/lotus#11422](https://github.com/filecoin-project/lotus/pull/11422))
+- chore: update bootstrap nodes ([filecoin-project/lotus#11288](https://github.com/filecoin-project/lotus/pull/11288))
+- chore: add bootstrap node on calibration ([filecoin-project/lotus#11175](https://github.com/filecoin-project/lotus/pull/11175))
+
+# 1.24.0 / 2023-11-22
+
+This is the stable release for the upcoming **MANDATORY**  Filecoin network upgrade v21, codenamed Watermelon 🍉, at **epoch 3469380 - 2023-12-12T13:30:00Z**.
+
+
+The Filecoin network version 21 delivers the following FIPs:
+
+- [FIP0052: Increase Max Sector Commitment to 3.5 years](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0052.md)
+- [FIP0059: Synthetic PoRep](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0059.md)
+- [FIP0071: Deterministic State Access (IPLD Reachability)](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0071.md)
+- [FIP0072: Improved event syscall API](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0072.md) 
+- [FIP0073: Remove beneficiary from the self_destruct syscall](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0073.md)
+- [FIP0075: Improvements to FVM randomness syscalls](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0075.md)
+
+
+Full list of the other protocol improvements we are delivering can be found [here](https://github.com/filecoin-project/core-devs/blob/master/Network%20Upgrades/v21.md).
+
+## ☢️ Upgrade Warnings ☢️
+
+This feature release requires a minimum Go version of v1.20.7 or higher to successfully build Lotus. Go version 1.21.x is not supported yet.
+
+## v12 Builtin Actor Bundles
+
+[Builtin actor v12.0.0](https://github.com/filecoin-project/builtin-actors/releases/tag/v12.0.0) is used for supporting this upgrade.
+Make sure that your lotus actor bundle matches the v12 actors manifest by running the following cli after upgrading:
+
+```
+./lotus state actor-cids --network-version 21
+Network Version: 21
+Actor Version: 12
+Manifest CID: bafy2bzaceapkgfggvxyllnmuogtwasmsv5qi2qzhc2aybockd6kag2g5lzaio
+
+Actor             CID  
+datacap           bafk2bzacebpiwb2ml4qbnnaayxumtk43ryhc63exdgnhivy3hwgmzemawsmpq
+paymentchannel    bafk2bzacectv4cm47bnhga5febf3lo3fq47g72kmmp2xd5s6tcxz7hiqdywa4
+storagemarket     bafk2bzacedylkg5am446lcuih4voyzdn4yjeqfsxfzh5b6mcuhx4mok5ph5c4
+storagepower      bafk2bzacecsij5tpfzjpfuckxvccv2p3bdqjklkrfyyoei6lx5dyj5j4fvjm6
+cron              bafk2bzacechxjkfe2cehx4s7skj3wzfpzf7zolds64khrrrs66bhazsemktls
+eam               bafk2bzaceb3elj4hfbbjp7g5bptc7su7mptszl4nlqfedilxvstjo5ungm6oe
+ethaccount        bafk2bzaceb4gkau2vgsijcxpfuq33bd7w3efr2rrhxrwiacjmns2ntdiamswq
+reward            bafk2bzacealqnxn5lwzwexd6reav4dppypquklx2ujlnvaxiqk2tzstyvkp5u
+verifiedregistry  bafk2bzacedudgflxc75c77c6zkmfyq4u2xuk7k6xw6dfdccarjrvxx453b77q
+evm               bafk2bzacecmnyfiwb52tkbwmm2dsd7ysi3nvuxl3lmspy7pl26wxj4zj7w4wi
+placeholder       bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro
+storageminer      bafk2bzacedo75pabe4i2l3hvhtsjmijrcytd2y76xwe573uku25fi7sugqld6
+system            bafk2bzacebfqrja2hip7esf4eafxjmu6xcogoqu5xxtgdg7xa5szgvvdguchu
+account           bafk2bzaceboftg75mdiba7xbo2i3uvgtca4brhnr3u5ptihonixgpnrvhpxoa
+init              bafk2bzacebllyegx5r6lggf6ymyetbp7amacwpuxakhtjvjtvoy2bfkzk3vms
+```
+
+## Migration  
+
+We are expecting a heavier than normal state migration for this upgrade due to the amount of state changes introduced for miner sector info. (This is a similar migration as the Shark upgrade, however, we have introduced a couple of migration performance optimizations since then for a smoother upgrade experience.)
+
+All node operators, including storage providers, should be aware that ONE pre-migration is being scheduled 180 epochs before the upgrade, around 2023-12-12T12:00:00Z. It will take around 20-30 minutes for the pre-migration and less than 30 seconds for the final migration, depending on the amount of historical state in the node blockstore and the hardware specs the node is running on. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries (during our testing, it topped out about 205 RAM(htop) on a 1TiB box).
+
+We recommend node operators (who haven't enabled splitstore `discard` mode) that do not care about historical chain states, to prune the chain blockstore by syncing from a snapshot 1-2 days before the upgrade.
+
+Note to full archival node operators: you may expect it takes some time  for the node to complete the final migration, during this period your node will fall out of sync and your chain service may have some disruption. However, you can expect the node to catch up soon after the migration completes. You can test out the migration by running the following on your node in offline mode:
+
+1. `lotus chain head | head -n1`
+2. Stop Lotus daemon
+3. `./lotus-shed migrate-state --repo=[path-to-your-lotus-repo] 21 [output-of-step-1]`
+
+You can check out the [tutorial for benchmarking the network migration here.](https://lotus.filecoin.io/kb/test-migration/)
+
+
+## BREAKING CHANGE
+
+There is a new protocol limit on how many partition could be submited in one PoSt - if you have any customized tooling for batching PoSts, please update accordingly. 
+- feat: limit PoSted partitions to 3 ([filecoin-project/lotus#11327](https://github.com/filecoin-project/lotus/pull/11327))
+
+## New features
+- Implement and support [FIP0052: Increase Max Sector Commitment to 3.5 years](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0052.md)
+   - fix: docs: Update SectorLifetime to be in line with FIP-0052 ([filecoin-project/lotus#11314](https://github.com/filecoin-project/lotus/pull/11314))
+- Implement and support [FIP0059: Synthetic PoRep](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0059.md) - Check out the [Lotus documentation for Synthetic PoRep](https://lotus.filecoin.io/storage-providers/advanced-configurations/sealing/#synthetic-porep).
+   - feat: implement Synthetic PoRep ([filecoin-project/lotus#11258](https://github.com/filecoin-project/lotus/pull/11258))
+   - chore: config: Update todo in UseSyntheticPoRep ([filecoin-project/lotus#11297](https://github.com/filecoin-project/lotus/pull/11297))
+   
+## Improvements
+- Backport: feat: sealing: Switch to calling PreCommitSectorBatch2 ([filecoin-project/lotus#11215](https://github.com/filecoin-project/lotus/pull/11215))
+- updated the boostrap nodes 
+
+## Dependencies
+- github.com/filecoin-project/go-amt-ipld/v4 (v4.0.0 -> v4.2.0)
+- chore: deps: update FFI, FVM, and actors ([filecoin-project/lotus#11310](https://github.com/filecoin-project/lotus/pull/11310))
+- chore: deps: update to final actors ([filecoin-project/lotus#11330](https://github.com/filecoin-project/lotus/pull/11440))
+- chore: deps: update to go-state-types v0.12.8 ([filecoin-project/lotus#11339](https://github.com/filecoin-project/lotus/pull/11437))
+- chore: deps: update libp2p to v0.30.0 #11434
+
+
+## Snapshots 
+
+The [Forest team](https://filecoinproject.slack.com/archives/C029LPZ5N73) at Chainsafe has launched a brand new lightweight snapshot service that is backed up by forest nodes! This is a great alternative service along with the fil-infra one, and it is compatible with lotus! We recommend lotus users to check it out [here](https://docs.filecoin.io/networks/mainnet#resources)! 
+
+
 
 # v1.23.3 / 2023-08-01
 
@@ -30,7 +262,7 @@ This feature release requires a **minimum Go version of v1.19.12 or higher to su
   - feat: sealing: flag to run data_cid untied from addpiece ([filecoin-project/lotus#10797](https://github.com/filecoin-project/lotus/pull/10797))
   - feat: Lotus Gateway: add MpoolPending, ChainGetBlock and MinerGetBaseInfo ([filecoin-project/lotus#10929](https://github.com/filecoin-project/lotus/pull/10929))
 
-## Improvements
+## Improvements && Bug Fixe
   - chore: update ffi & fvm ([filecoin-project/lotus#11040](https://github.com/filecoin-project/lotus/pull/11040))
   - feat: Make sure we don't store duplidate actor events caused to reorgs in events.db ([filecoin-project/lotus#11015](https://github.com/filecoin-project/lotus/pull/11015))
   - sealing: Use only non-assigned deals when selecting snap sectors ([filecoin-project/lotus#11002](https://github.com/filecoin-project/lotus/pull/11002))
@@ -108,6 +340,10 @@ This feature release requires a **minimum Go version of v1.19.12 or higher to su
   - fix: cli: Change arg wording in change-beneficiary cmd ([filecoin-project/lotus#10823](https://github.com/filecoin-project/lotus/pull/10823))
   - refactor: streamline error handling in CheckPendingMessages (#10818) ([filecoin-project/lotus#10818](https://github.com/filecoin-project/lotus/pull/10818))
   - feat: Add tmp indices to events table while performing migration to V2
+  - fix: sync: iterate over returned messages directly #11373
+  - fix: api: compute the effective gas cost with the correct base-fee #11357
+  - fix: check invariants: v12 check #11371
+  - fix: api: compute gasUsedRatio based on max gas in the tipset #11354
 
 # v1.23.2 / 2023-06-28
 

Dockerfile

@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.19.12-bullseye AS lotus-builder
+FROM golang:1.20.7-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

GO_VERSION_MIN

@@ -1 +1 @@
-1.19.12
+1.20.7

Makefile

@@ -193,7 +193,7 @@ lotus-health:
 .PHONY: lotus-health
 BINS+=lotus-health
 
-lotus-wallet:
+lotus-wallet: $(BUILD_DEPS)
 	rm -f lotus-wallet
 	$(GOCC) build $(GOFLAGS) -o lotus-wallet ./cmd/lotus-wallet
 .PHONY: lotus-wallet

api/api_full.go

@@ -824,7 +824,7 @@ type FullNode interface {
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)                                                                          //perm:read
 
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)                                             //perm:read
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)                                 //perm:read
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)                                 //perm:read
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
 
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read
@@ -867,6 +867,13 @@ type FullNode interface {
 	// Returns the client version
 	Web3ClientVersion(ctx context.Context) (string, error) //perm:read
 
+	// TraceAPI related methods
+	//
+	// Returns traces created at given block
+	EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error) //perm:read
+	// Replays all transactions in a block returning the requested traces for each transaction
+	EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) //perm:read
+
 	// CreateBackup creates node backup onder the specified file name. The
 	// method requires that the lotus daemon is running with the
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that

api/api_gateway.go

@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
+	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/dline"
 
 	apitypes "github.com/filecoin-project/lotus/api/types"
@@ -65,6 +66,11 @@ type Gateway interface {
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
+	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
+	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
+	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
 	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
@@ -108,7 +114,7 @@ type Gateway interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)
@@ -121,4 +127,6 @@ type Gateway interface {
 	EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error)
 	EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error)
 	Web3ClientVersion(ctx context.Context) (string, error)
+	EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error)
+	EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error)
 }

api/eth_aliases.go

@@ -40,6 +40,9 @@ func CreateEthRPCAliases(as apitypes.Aliaser) {
 	as.AliasMethod("eth_subscribe", "Filecoin.EthSubscribe")
 	as.AliasMethod("eth_unsubscribe", "Filecoin.EthUnsubscribe")
 
+	as.AliasMethod("trace_block", "Filecoin.EthTraceBlock")
+	as.AliasMethod("trace_replayBlockTransactions", "Filecoin.EthTraceReplayBlockTransactions")
+
 	as.AliasMethod("net_version", "Filecoin.NetVersion")
 	as.AliasMethod("net_listening", "Filecoin.NetListening")
 

api/mocks/mock_full.go

@@ -1042,7 +1042,7 @@ func (mr *MockFullNodeMockRecorder) EthChainId(arg0 interface{}) *gomock.Call {
 }
 
 // EthEstimateGas mocks base method.
-func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "EthEstimateGas", arg0, arg1)
 	ret0, _ := ret[0].(ethtypes.EthUint64)
@@ -1491,6 +1491,36 @@ func (mr *MockFullNodeMockRecorder) EthSyncing(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthSyncing", reflect.TypeOf((*MockFullNode)(nil).EthSyncing), arg0)
 }
 
+// EthTraceBlock mocks base method.
+func (m *MockFullNode) EthTraceBlock(arg0 context.Context, arg1 string) ([]*ethtypes.EthTraceBlock, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EthTraceBlock", arg0, arg1)
+	ret0, _ := ret[0].([]*ethtypes.EthTraceBlock)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// EthTraceBlock indicates an expected call of EthTraceBlock.
+func (mr *MockFullNodeMockRecorder) EthTraceBlock(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthTraceBlock", reflect.TypeOf((*MockFullNode)(nil).EthTraceBlock), arg0, arg1)
+}
+
+// EthTraceReplayBlockTransactions mocks base method.
+func (m *MockFullNode) EthTraceReplayBlockTransactions(arg0 context.Context, arg1 string, arg2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "EthTraceReplayBlockTransactions", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]*ethtypes.EthTraceReplayBlockTransaction)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// EthTraceReplayBlockTransactions indicates an expected call of EthTraceReplayBlockTransactions.
+func (mr *MockFullNodeMockRecorder) EthTraceReplayBlockTransactions(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthTraceReplayBlockTransactions", reflect.TypeOf((*MockFullNode)(nil).EthTraceReplayBlockTransactions), arg0, arg1, arg2)
+}
+
 // EthUninstallFilter mocks base method.
 func (m *MockFullNode) EthUninstallFilter(arg0 context.Context, arg1 ethtypes.EthFilterID) (bool, error) {
 	m.ctrl.T.Helper()

api/proxy_gen.go

@@ -255,7 +255,7 @@ type FullNodeMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) `perm:"read"`
 
-	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) `perm:"read"`
+	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) `perm:"read"`
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) `perm:"read"`
 
@@ -315,6 +315,10 @@ type FullNodeMethods struct {
 
 	EthSyncing func(p0 context.Context) (ethtypes.EthSyncingResult, error) `perm:"read"`
 
+	EthTraceBlock func(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) `perm:"read"`
+
+	EthTraceReplayBlockTransactions func(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) `perm:"read"`
+
 	EthUninstallFilter func(p0 context.Context, p1 ethtypes.EthFilterID) (bool, error) `perm:"read"`
 
 	EthUnsubscribe func(p0 context.Context, p1 ethtypes.EthSubscriptionID) (bool, error) `perm:"read"`
@@ -675,7 +679,7 @@ type GatewayMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) ``
 
-	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) ``
+	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) ``
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) ``
 
@@ -731,6 +735,10 @@ type GatewayMethods struct {
 
 	EthSyncing func(p0 context.Context) (ethtypes.EthSyncingResult, error) ``
 
+	EthTraceBlock func(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) ``
+
+	EthTraceReplayBlockTransactions func(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) ``
+
 	EthUninstallFilter func(p0 context.Context, p1 ethtypes.EthFilterID) (bool, error) ``
 
 	EthUnsubscribe func(p0 context.Context, p1 ethtypes.EthSubscriptionID) (bool, error) ``
@@ -769,6 +777,16 @@ type GatewayMethods struct {
 
 	StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) ``
 
+	StateGetAllocation func(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) ``
+
+	StateGetAllocationForPendingDeal func(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) ``
+
+	StateGetAllocations func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) ``
+
+	StateGetClaim func(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) ``
+
+	StateGetClaims func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) ``
+
 	StateListMiners func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) ``
 
 	StateLookupID func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
@@ -2116,14 +2134,14 @@ func (s *FullNodeStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
@@ -2446,6 +2464,28 @@ func (s *FullNodeStub) EthSyncing(p0 context.Context) (ethtypes.EthSyncingResult
 	return *new(ethtypes.EthSyncingResult), ErrNotSupported
 }
 
+func (s *FullNodeStruct) EthTraceBlock(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) {
+	if s.Internal.EthTraceBlock == nil {
+		return *new([]*ethtypes.EthTraceBlock), ErrNotSupported
+	}
+	return s.Internal.EthTraceBlock(p0, p1)
+}
+
+func (s *FullNodeStub) EthTraceBlock(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) {
+	return *new([]*ethtypes.EthTraceBlock), ErrNotSupported
+}
+
+func (s *FullNodeStruct) EthTraceReplayBlockTransactions(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	if s.Internal.EthTraceReplayBlockTransactions == nil {
+		return *new([]*ethtypes.EthTraceReplayBlockTransaction), ErrNotSupported
+	}
+	return s.Internal.EthTraceReplayBlockTransactions(p0, p1, p2)
+}
+
+func (s *FullNodeStub) EthTraceReplayBlockTransactions(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	return *new([]*ethtypes.EthTraceReplayBlockTransaction), ErrNotSupported
+}
+
 func (s *FullNodeStruct) EthUninstallFilter(p0 context.Context, p1 ethtypes.EthFilterID) (bool, error) {
 	if s.Internal.EthUninstallFilter == nil {
 		return false, ErrNotSupported
@@ -4360,14 +4400,14 @@ func (s *GatewayStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error)
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
@@ -4668,6 +4708,28 @@ func (s *GatewayStub) EthSyncing(p0 context.Context) (ethtypes.EthSyncingResult,
 	return *new(ethtypes.EthSyncingResult), ErrNotSupported
 }
 
+func (s *GatewayStruct) EthTraceBlock(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) {
+	if s.Internal.EthTraceBlock == nil {
+		return *new([]*ethtypes.EthTraceBlock), ErrNotSupported
+	}
+	return s.Internal.EthTraceBlock(p0, p1)
+}
+
+func (s *GatewayStub) EthTraceBlock(p0 context.Context, p1 string) ([]*ethtypes.EthTraceBlock, error) {
+	return *new([]*ethtypes.EthTraceBlock), ErrNotSupported
+}
+
+func (s *GatewayStruct) EthTraceReplayBlockTransactions(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	if s.Internal.EthTraceReplayBlockTransactions == nil {
+		return *new([]*ethtypes.EthTraceReplayBlockTransaction), ErrNotSupported
+	}
+	return s.Internal.EthTraceReplayBlockTransactions(p0, p1, p2)
+}
+
+func (s *GatewayStub) EthTraceReplayBlockTransactions(p0 context.Context, p1 string, p2 []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	return *new([]*ethtypes.EthTraceReplayBlockTransaction), ErrNotSupported
+}
+
 func (s *GatewayStruct) EthUninstallFilter(p0 context.Context, p1 ethtypes.EthFilterID) (bool, error) {
 	if s.Internal.EthUninstallFilter == nil {
 		return false, ErrNotSupported
@@ -4877,6 +4939,61 @@ func (s *GatewayStub) StateGetActor(p0 context.Context, p1 address.Address, p2 t
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) StateGetAllocation(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocation == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetAllocation(p0, p1, p2, p3)
+}
+
+func (s *GatewayStub) StateGetAllocation(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetAllocationForPendingDeal(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocationForPendingDeal == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetAllocationForPendingDeal(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetAllocationForPendingDeal(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetAllocations(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocations == nil {
+		return *new(map[verifregtypes.AllocationId]verifregtypes.Allocation), ErrNotSupported
+	}
+	return s.Internal.StateGetAllocations(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetAllocations(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) {
+	return *new(map[verifregtypes.AllocationId]verifregtypes.Allocation), ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetClaim(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) {
+	if s.Internal.StateGetClaim == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetClaim(p0, p1, p2, p3)
+}
+
+func (s *GatewayStub) StateGetClaim(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetClaims(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) {
+	if s.Internal.StateGetClaims == nil {
+		return *new(map[verifregtypes.ClaimId]verifregtypes.Claim), ErrNotSupported
+	}
+	return s.Internal.StateGetClaims(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetClaims(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) {
+	return *new(map[verifregtypes.ClaimId]verifregtypes.Claim), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateListMiners(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) {
 	if s.Internal.StateListMiners == nil {
 		return *new([]address.Address), ErrNotSupported

api/types.go

@@ -312,6 +312,7 @@ type NetworkParams struct {
 	SupportedProofTypes     []abi.RegisteredSealProof
 	PreCommitChallengeDelay abi.ChainEpoch
 	ForkUpgradeParams       ForkUpgradeParams
+	Eip155ChainID           int
 }
 
 type ForkUpgradeParams struct {

api/v0api/gateway.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/dline"
 	abinetwork "github.com/filecoin-project/go-state-types/network"
 
@@ -61,6 +62,11 @@ type Gateway interface {
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
 	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
+	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
+	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
+	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
 	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)

api/v0api/proxy_gen.go

@@ -478,6 +478,16 @@ type GatewayMethods struct {
 
 	StateGetActor func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*types.Actor, error) ``
 
+	StateGetAllocation func(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) ``
+
+	StateGetAllocationForPendingDeal func(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) ``
+
+	StateGetAllocations func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) ``
+
+	StateGetClaim func(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) ``
+
+	StateGetClaims func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) ``
+
 	StateGetReceipt func(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) ``
 
 	StateListMiners func(p0 context.Context, p1 types.TipSetKey) ([]address.Address, error) ``
@@ -2850,6 +2860,61 @@ func (s *GatewayStub) StateGetActor(p0 context.Context, p1 address.Address, p2 t
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) StateGetAllocation(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocation == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetAllocation(p0, p1, p2, p3)
+}
+
+func (s *GatewayStub) StateGetAllocation(p0 context.Context, p1 address.Address, p2 verifregtypes.AllocationId, p3 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetAllocationForPendingDeal(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocationForPendingDeal == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetAllocationForPendingDeal(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetAllocationForPendingDeal(p0 context.Context, p1 abi.DealID, p2 types.TipSetKey) (*verifregtypes.Allocation, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetAllocations(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) {
+	if s.Internal.StateGetAllocations == nil {
+		return *new(map[verifregtypes.AllocationId]verifregtypes.Allocation), ErrNotSupported
+	}
+	return s.Internal.StateGetAllocations(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetAllocations(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error) {
+	return *new(map[verifregtypes.AllocationId]verifregtypes.Allocation), ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetClaim(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) {
+	if s.Internal.StateGetClaim == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateGetClaim(p0, p1, p2, p3)
+}
+
+func (s *GatewayStub) StateGetClaim(p0 context.Context, p1 address.Address, p2 verifregtypes.ClaimId, p3 types.TipSetKey) (*verifregtypes.Claim, error) {
+	return nil, ErrNotSupported
+}
+
+func (s *GatewayStruct) StateGetClaims(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) {
+	if s.Internal.StateGetClaims == nil {
+		return *new(map[verifregtypes.ClaimId]verifregtypes.Claim), ErrNotSupported
+	}
+	return s.Internal.StateGetClaims(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateGetClaims(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error) {
+	return *new(map[verifregtypes.ClaimId]verifregtypes.Claim), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateGetReceipt(p0 context.Context, p1 cid.Cid, p2 types.TipSetKey) (*types.MessageReceipt, error) {
 	if s.Internal.StateGetReceipt == nil {
 		return nil, ErrNotSupported

build/bootstrap/butterflynet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWRaoPgwJuZdPSN4A2iTeh8xzkZGCEBxan9vMkidHisUgn
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWMjLCZeEf3VzSWvQYuhe9VzCcrN6RENX9FgmQqiJfQDWs
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWNwAkUtWuLtKCyyFP2vBzmpTHSrQao7KQx7Xfa8YvSg1N
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWPn8BDeNcctAcAGuxxiic8uMw2pAi3G5vgdFtfgRs5zBu

build/bootstrap/calibnet.pi

@@ -2,3 +2,4 @@
 /dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWDTayrBojBn9jWNNUih4nNQQBGJD7Zo3gQCKgBkUsS6dp
 /dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWNRxTHUn8bf7jz1KEUPMc2dMgGfa4f8ZJTsquVSn3vHCG
 /dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWFWUqE9jgXvcKHWieYs9nhyp6NF4ftwLGAHm4sCv73jjK
+/dns4/calibration.node.glif.io/tcp/1237/p2p/12D3KooWQPYouEAsUQKzvFUA9sQ8tz4rfpqtTzh2eL6USd9bwg7x

build/bootstrap/mainnet.pi

@@ -13,4 +13,5 @@
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
 /dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
 /dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
-/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
+/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
+/dns4/bootstrap-venus.mainnet.filincubator.com/tcp/8888/p2p/QmQu8C6deXwKvJP2D8B6QGyhngc3ZiDnFzEHBDx8yeBXST

build/builtin_actors.go

@@ -42,6 +42,13 @@ func init() {
 	if err := loadManifests(NetworkBundle); err != nil {
 		panic(err)
 	}
+
+	// The following code cid existed temporarily on the calibnet testnet, as a "buggy" storage miner actor implementation.
+	// We include them in our builtin bundle, but intentionally omit from metadata.
+	if NetworkBundle == "calibrationnet" {
+		actors.AddActorMeta("storageminer", cid.MustParse("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq"), actorstypes.Version12)
+		actors.AddActorMeta("storageminer", cid.MustParse("bafk2bzaced7emkbbnrewv5uvrokxpf5tlm4jslu2jsv77ofw2yqdglg657uie"), actorstypes.Version12)
+	}
 }
 
 // UseNetworkBundle switches to a different network bundle, by name.
@@ -183,6 +190,13 @@ func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata,
 		if err != nil {
 			return nil, xerrors.Errorf("error loading builtin actors bundle: %w", err)
 		}
+
+		// The following manifest cids existed temporarily on the calibnet testnet
+		// We include them in our builtin bundle, but intentionally omit from metadata
+		if root == cid.MustParse("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs") ||
+			root == cid.MustParse("bafy2bzacebl4w5ptfvuw6746w7ev562idkbf5ppq72e6zub22435ws2rukzru") {
+			continue
+		}
 		bundles = append(bundles, &BuiltinActorsMetadata{
 			Network:     name,
 			Version:     actorstypes.Version(version),
@@ -232,7 +246,7 @@ func readBundleManifest(r io.Reader) (cid.Cid, map[string]cid.Cid, error) {
 }
 
 // GetEmbeddedBuiltinActorsBundle returns the builtin-actors bundle for the given actors version.
-func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version) ([]byte, bool) {
+func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version, networkBundleName string) ([]byte, bool) {
 	fi, err := embeddedBuiltinActorReleases.Open(fmt.Sprintf("actors/v%d.tar.zst", version))
 	if err != nil {
 		return nil, false
@@ -243,7 +257,7 @@ func GetEmbeddedBuiltinActorsBundle(version actorstypes.Version) ([]byte, bool)
 	defer uncompressed.Close() //nolint
 
 	tarReader := tar.NewReader(uncompressed)
-	targetFileName := fmt.Sprintf("builtin-actors-%s.car", NetworkBundle)
+	targetFileName := fmt.Sprintf("builtin-actors-%s.car", networkBundleName)
 	for {
 		header, err := tarReader.Next()
 		switch err {

build/builtin_actors_gen.go

@@ -97,8 +97,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "butterflynet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzaceb7s3k6zyuubqil7s2rwdcdyslqqx2fn2cj536yh3hexlyi4syyk6"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzacectxvbk77ntedhztd6sszp2btrtvsmy7lp2ypnrk6yl74zb34t2cq"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacebp7anjdtg2sohyt6lromx4xs7nujtwdfcsffnptphaayabx7ysxs"),
 		"cron":             MustParseCid("bafk2bzacecu2y3awtemmglpkroiglulc2fj3gpdn6eazdqr6avcautiaighrg"),
@@ -112,7 +112,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzacedebvitdsztwebi44t5es4ls3p3hor252igzawr3s6uznmbvzh2ou"),
 		"storagemarket":    MustParseCid("bafk2bzaceb2tdeqtt2eqpzeb3gezuchb7g7uzbd52bgvcdt6bg3ckq7oisb74"),
-		"storageminer":     MustParseCid("bafk2bzaceaaztuglvw6pw5xohwfekbw47iyugbyftxrp6hwygsaiu6rebtcty"),
+		"storageminer":     MustParseCid("bafk2bzaceb62clldtod2jimnri5k2koxttf6vqtlsvkjhnwduzs7sgsoakglw"),
 		"storagepower":     MustParseCid("bafk2bzacedxvlj5xmhytdjrjqyonz37duvxb2ioyzk75c27yypkqalxuh3xh6"),
 		"system":           MustParseCid("bafk2bzacec3vwj2chzaram3iqupkbfiein5h2l5qiltlrngbju2vg5umelclm"),
 		"verifiedregistry": MustParseCid("bafk2bzacedv2irkql7nil3w5v3ohqq3e54w62pxeoppjmaktzokolaaoh5ksu"),
@@ -203,8 +203,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "calibrationnet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzacednzb3pkrfnbfhmoqtb3bc6dgvxszpqklf3qcc7qzcage4ewzxsca"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacechwwxdqvggkdylm37zldjsra2ivkdzwp7fee56bzxbzs544wv6u6"),
 		"cron":             MustParseCid("bafk2bzacec4gdxxkqwxqqodsv6ug5dmdbqdfqwyqfek3yhxc2wweh5psxaeq6"),
@@ -218,7 +218,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzacedra77pcglf7vdca2itcaa4vd6xrxynxmgfgdjdxqxfwqyhtoxehy"),
 		"storagemarket":    MustParseCid("bafk2bzacea7g46y7xxu2zjq2h75x6mmx3utz2uxnlvnwi6tzpsvulna3bmiva"),
-		"storageminer":     MustParseCid("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq"),
+		"storageminer":     MustParseCid("bafk2bzaceb7qzqsi5uyxe4o5iuasi47l2hnznvmqr2eu4pl3qscvarjqlnuxo"),
 		"storagepower":     MustParseCid("bafk2bzacedd3ka44k7d46ckbinjhv3diyuu2epgbyvhqqyjkc64qlrg3wlgzi"),
 		"system":           MustParseCid("bafk2bzacecioupndtcnyw6iq2hbrxag3aufvczlv5nobnfbkbywqzcyfaa376"),
 		"verifiedregistry": MustParseCid("bafk2bzaceavldupmf7bimeeacs67z5xdfdlfca6p7sn6bev3mt5ggepfqvhqo"),
@@ -318,8 +318,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "caterpillarnet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacedrimdbavexkdqldjwczbhlx4k5byshl363z62rrt6vkt7mbvnulu"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzacebxiub6qsy67asvl5cx33x5vjbuqinalmf3xtnbmokxmmklzdkvei"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacecereuhejfvodut5357cai4lmhsyr7uenhcxvmw6jpmhe6auuly32"),
 		"cron":             MustParseCid("bafk2bzacebo2whgy6jla4jsf5j4ovlqm2e4eepedlpw5wadas33yxmunis4b4"),
@@ -333,7 +333,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzaceb2x5zgkrho373l3ippp6krs7brugssg6hj2tib22xmqjzdm2o25a"),
 		"storagemarket":    MustParseCid("bafk2bzaced5j6drzmsebpxbf2dtptrc5tyidlbftdljqxavxqb57s2qpbvdek"),
-		"storageminer":     MustParseCid("bafk2bzacedozwxc3c6nqqajafs3hsjviyijhthysd5two5ptm5fcdbwh224ag"),
+		"storageminer":     MustParseCid("bafk2bzaceckgusfenkczxolfczvnygnuhxbou5to2skwwngbkihla7hgdv4yy"),
 		"storagepower":     MustParseCid("bafk2bzaceagp6ilkltsltwii66nz6a4zen4qtfk7rdkvdv3gzq7fbv4ivox3u"),
 		"system":           MustParseCid("bafk2bzacedye5j5uxox7knb6zlnhseaadztyav76mjbyk5qslhhbpiy5cdtt2"),
 		"verifiedregistry": MustParseCid("bafk2bzacecduww5pirr7dvaijjijw4gf6ygf7vipgxh4scvv6vseo46gueb46"),
@@ -424,8 +424,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "devnet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacebk6yiirh4ennphzyka7b6g6jzn3lt4lr5ht7rjwulnrcthjihapo"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzaceasjdukhhyjbegpli247vbf5h64f7uvxhhebdihuqsj2mwisdwa6o"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacedki4apynvdxxuoigmqkgaktgy2erjftoxqxqaklnelgveyaqknfu"),
 		"cron":             MustParseCid("bafk2bzacebjpczf7qtcisy3zdp3sqoohxe75tgupmdo5dr26vh7orzrsjn3b2"),
@@ -439,7 +439,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzacebzso6xkjxdscbpncw7el2d4hap6lfkgwqzrbc76lzp33vkwk6obc"),
 		"storagemarket":    MustParseCid("bafk2bzacebzg74vyk3gzbhnz4zviwvxblyar574mtd6ayognmsvlkriejmunu"),
-		"storageminer":     MustParseCid("bafk2bzaceajgt523lr2sf6cacvzo3goyalljlkaoeehyhxlv57wevkljw2cps"),
+		"storageminer":     MustParseCid("bafk2bzacecs262232b3awcrilyzpdketeayyqzzwgoavtxilgjvayrz55ovk4"),
 		"storagepower":     MustParseCid("bafk2bzacebbtj2m2ajawfuzxqz5nmdep7xevjo2qfjqa5tx3vr5m6qojolya4"),
 		"system":           MustParseCid("bafk2bzacecnau5wddulbsvwn75tc3w75jrlvkybgrlxs4ngonqab6xq3eowvg"),
 		"verifiedregistry": MustParseCid("bafk2bzacec37mddea65nvh4htsagtryfa3sq6i67utcupslyhzbhjhoy6hopa"),
@@ -553,8 +553,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "mainnet",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacebxmnxdgt2usdrcrpotgodr75culawwv4eklaun4lu6vlxecql6h2"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzaceapkgfggvxyllnmuogtwasmsv5qi2qzhc2aybockd6kag2g5lzaio"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzaceboftg75mdiba7xbo2i3uvgtca4brhnr3u5ptihonixgpnrvhpxoa"),
 		"cron":             MustParseCid("bafk2bzacechxjkfe2cehx4s7skj3wzfpzf7zolds64khrrrs66bhazsemktls"),
@@ -568,7 +568,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzacealqnxn5lwzwexd6reav4dppypquklx2ujlnvaxiqk2tzstyvkp5u"),
 		"storagemarket":    MustParseCid("bafk2bzacedylkg5am446lcuih4voyzdn4yjeqfsxfzh5b6mcuhx4mok5ph5c4"),
-		"storageminer":     MustParseCid("bafk2bzacea56yi2khvvorofkf2t4rb5fbmltahrjtauszirdjz4jbunt55xsk"),
+		"storageminer":     MustParseCid("bafk2bzacedo75pabe4i2l3hvhtsjmijrcytd2y76xwe573uku25fi7sugqld6"),
 		"storagepower":     MustParseCid("bafk2bzacecsij5tpfzjpfuckxvccv2p3bdqjklkrfyyoei6lx5dyj5j4fvjm6"),
 		"system":           MustParseCid("bafk2bzacebfqrja2hip7esf4eafxjmu6xcogoqu5xxtgdg7xa5szgvvdguchu"),
 		"verifiedregistry": MustParseCid("bafk2bzacedudgflxc75c77c6zkmfyq4u2xuk7k6xw6dfdccarjrvxx453b77q"),
@@ -659,8 +659,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "testing",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacebzcqzzlb5iwrledyuyy7pvob3xwzoxqsrvrbrnpaiejb52na5eza"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzaceaaxd6ytavsek5bi5soqo7qamezuqfyfjy42es2clpbzu3pwzcmye"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacea74qqkfvacykmq5emzqblh4f4nmxdkiyixxpzs7kkcfnbfa7cb6m"),
 		"cron":             MustParseCid("bafk2bzacecotbu7k6awdzfzakf7g5iaas6gswtunjnnb2xm2klqoshjgb4imy"),
@@ -674,7 +674,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzaceaajyncjxcrl7wbb6ukzkueyqz4uyekxpmtn4mpndkf7ksmggopzq"),
 		"storagemarket":    MustParseCid("bafk2bzaced6cexdcinwjhtfvuxgkxukiejp3goylaxtvhqfd24rs5z7g2z7dm"),
-		"storageminer":     MustParseCid("bafk2bzacecw6jvvfu62jkwsa4vzmn262cjrrxh6wza6nvjsyg5aeix7rzgmbe"),
+		"storageminer":     MustParseCid("bafk2bzacecvkbsjhufq2zr2dojohukdnql3gkqzdkmtp2hxvn5kczxp3tu6ko"),
 		"storagepower":     MustParseCid("bafk2bzacedexrf5qplrrl5xzijfrthjdqwodfs5e6zj5kpztc7qnywbqdyiii"),
 		"system":           MustParseCid("bafk2bzacecp4roanbxq3bflftlkipsoqqxio5etjjnzxus5pcu7lq43fnxb34"),
 		"verifiedregistry": MustParseCid("bafk2bzaceandytrgcnuvizfi47sijbqh6c243vjtzlzumexm6kjv7s7hye45g"),
@@ -765,8 +765,8 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 }, {
 	Network:      "testing-fake-proofs",
 	Version:      12,
-	BundleGitTag: "v12.0.0-rc.1",
-	ManifestCid:  MustParseCid("bafy2bzacedtskkmrkkzairdi66syrmvfpydg63ro4dlmmqeta5krnur4hcp3a"),
+	BundleGitTag: "v12.0.0",
+	ManifestCid:  MustParseCid("bafy2bzacecver4l5d6jiuzubhrtcxjjfdx6jnxbmyp4bselol2atgkhz3e3um"),
 	Actors: map[string]cid.Cid{
 		"account":          MustParseCid("bafk2bzacea74qqkfvacykmq5emzqblh4f4nmxdkiyixxpzs7kkcfnbfa7cb6m"),
 		"cron":             MustParseCid("bafk2bzacecotbu7k6awdzfzakf7g5iaas6gswtunjnnb2xm2klqoshjgb4imy"),
@@ -780,7 +780,7 @@ var EmbeddedBuiltinActorsMetadata []*BuiltinActorsMetadata = []*BuiltinActorsMet
 		"placeholder":      MustParseCid("bafk2bzacedfvut2myeleyq67fljcrw4kkmn5pb5dpyozovj7jpoez5irnc3ro"),
 		"reward":           MustParseCid("bafk2bzaceaajyncjxcrl7wbb6ukzkueyqz4uyekxpmtn4mpndkf7ksmggopzq"),
 		"storagemarket":    MustParseCid("bafk2bzaced6cexdcinwjhtfvuxgkxukiejp3goylaxtvhqfd24rs5z7g2z7dm"),
-		"storageminer":     MustParseCid("bafk2bzacea54x727xteh3tl46mitigy6cldqusqjxiwni65wrpofidh62k2ds"),
+		"storageminer":     MustParseCid("bafk2bzacedapzgrbc2rsmuqex76ftt2b62q6opi56gh2dr2oyyzuwin62rweg"),
 		"storagepower":     MustParseCid("bafk2bzacecdwijcbbryinjtm27pdinqqkyzoskri24pwsvsadwcq2alkkjpnc"),
 		"system":           MustParseCid("bafk2bzacecp4roanbxq3bflftlkipsoqqxio5etjjnzxus5pcu7lq43fnxb34"),
 		"verifiedregistry": MustParseCid("bafk2bzaceandytrgcnuvizfi47sijbqh6c243vjtzlzumexm6kjv7s7hye45g"),

build/params_2k.go

@@ -67,6 +67,12 @@ var UpgradeThunderHeight = abi.ChainEpoch(-23)
 
 var UpgradeWatermelonHeight = abi.ChainEpoch(200)
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -100
+
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFix2Height = -101
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
@@ -132,6 +138,8 @@ const BlockDelaySecs = uint64(4)
 
 const PropagationDelaySecs = uint64(1)
 
+var EquivocationDelaySecs = uint64(0)
+
 // SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
 // which the miner is slashed
 //

build/params_butterfly.go

@@ -57,6 +57,12 @@ const UpgradeThunderHeight = -23
 
 const UpgradeWatermelonHeight = 400
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -100
+
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFix2Height = -101
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg512MiBV1,
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
@@ -83,6 +89,8 @@ const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
 
 const PropagationDelaySecs = uint64(6)
 
+var EquivocationDelaySecs = uint64(2)
+
 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 2
 

build/params_calibnet.go

@@ -82,6 +82,12 @@ const UpgradeThunderHeight = UpgradeLightningHeight + 3120
 // 2023-10-19T13:00:00Z
 const UpgradeWatermelonHeight = 1013134
 
+// 2023-11-07T13:00:00Z
+const UpgradeWatermelonFixHeight = 1070494
+
+// 2023-11-21T13:00:00Z
+const UpgradeWatermelonFix2Height = 1108174
+
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
 	abi.RegisteredSealProof_StackedDrg64GiBV1,
@@ -123,6 +129,8 @@ const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
 
 var PropagationDelaySecs = uint64(10)
 
+var EquivocationDelaySecs = uint64(2)
+
 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 4
 

build/params_interop.go

@@ -56,6 +56,12 @@ var UpgradeThunderHeight = abi.ChainEpoch(-23)
 
 const UpgradeWatermelonHeight = 50
 
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -1
+
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFix2Height = -2
+
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,
 }
@@ -121,6 +127,8 @@ const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
 
 const PropagationDelaySecs = uint64(6)
 
+var EquivocationDelaySecs = uint64(2)
+
 // BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
 const BootstrapPeerThreshold = 2
 

build/params_mainnet.go

@@ -95,8 +95,14 @@ const UpgradeLightningHeight = 2809800
 // 2023-05-18T13:00:00Z
 const UpgradeThunderHeight = UpgradeLightningHeight + 2880*21
 
-// ???????
-var UpgradeWatermelonHeight = abi.ChainEpoch(9999999999)
+// 2023-12-12T13:30:00Z
+var UpgradeWatermelonHeight = abi.ChainEpoch(3469380)
+
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFixHeight = -1
+
+// This fix upgrade only ran on calibrationnet
+const UpgradeWatermelonFix2Height = -2
 
 var SupportedProofTypes = []abi.RegisteredSealProof{
 	abi.RegisteredSealProof_StackedDrg32GiBV1,
@@ -106,6 +112,8 @@ var ConsensusMinerMinPower = abi.NewStoragePower(10 << 40)
 var PreCommitChallengeDelay = abi.ChainEpoch(150)
 var PropagationDelaySecs = uint64(10)
 
+var EquivocationDelaySecs = uint64(2)
+
 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
 		SetAddressNetwork(address.Mainnet)

build/params_testground.go

@@ -9,7 +9,6 @@ package build
 
 import (
 	"math/big"
-	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -34,6 +33,7 @@ var (
 	MinimumBaseFee        = int64(100)
 	BlockDelaySecs        = uint64(builtin2.EpochDurationSeconds)
 	PropagationDelaySecs  = uint64(6)
+	EquivocationDelaySecs = uint64(2)
 	SupportedProofTypes   = []abi.RegisteredSealProof{
 		abi.RegisteredSealProof_StackedDrg32GiBV1,
 		abi.RegisteredSealProof_StackedDrg64GiBV1,
@@ -87,29 +87,31 @@ var (
 	UpgradeBreezeHeight      abi.ChainEpoch = -1
 	BreezeGasTampingDuration abi.ChainEpoch = 0
 
-	UpgradeSmokeHeight      abi.ChainEpoch = -1
-	UpgradeIgnitionHeight   abi.ChainEpoch = -2
-	UpgradeRefuelHeight     abi.ChainEpoch = -3
-	UpgradeTapeHeight       abi.ChainEpoch = -4
-	UpgradeAssemblyHeight   abi.ChainEpoch = 10
-	UpgradeLiftoffHeight    abi.ChainEpoch = -5
-	UpgradeKumquatHeight    abi.ChainEpoch = -6
-	UpgradeCalicoHeight     abi.ChainEpoch = -8
-	UpgradePersianHeight    abi.ChainEpoch = -9
-	UpgradeOrangeHeight     abi.ChainEpoch = -10
-	UpgradeClausHeight      abi.ChainEpoch = -11
-	UpgradeTrustHeight      abi.ChainEpoch = -12
-	UpgradeNorwegianHeight  abi.ChainEpoch = -13
-	UpgradeTurboHeight      abi.ChainEpoch = -14
-	UpgradeHyperdriveHeight abi.ChainEpoch = -15
-	UpgradeChocolateHeight  abi.ChainEpoch = -16
-	UpgradeOhSnapHeight     abi.ChainEpoch = -17
-	UpgradeSkyrHeight       abi.ChainEpoch = -18
-	UpgradeSharkHeight      abi.ChainEpoch = -19
-	UpgradeHyggeHeight      abi.ChainEpoch = -20
-	UpgradeLightningHeight  abi.ChainEpoch = -21
-	UpgradeThunderHeight    abi.ChainEpoch = -22
-	UpgradeWatermelonHeight abi.ChainEpoch = -23
+	UpgradeSmokeHeight          abi.ChainEpoch = -1
+	UpgradeIgnitionHeight       abi.ChainEpoch = -2
+	UpgradeRefuelHeight         abi.ChainEpoch = -3
+	UpgradeTapeHeight           abi.ChainEpoch = -4
+	UpgradeAssemblyHeight       abi.ChainEpoch = 10
+	UpgradeLiftoffHeight        abi.ChainEpoch = -5
+	UpgradeKumquatHeight        abi.ChainEpoch = -6
+	UpgradeCalicoHeight         abi.ChainEpoch = -8
+	UpgradePersianHeight        abi.ChainEpoch = -9
+	UpgradeOrangeHeight         abi.ChainEpoch = -10
+	UpgradeClausHeight          abi.ChainEpoch = -11
+	UpgradeTrustHeight          abi.ChainEpoch = -12
+	UpgradeNorwegianHeight      abi.ChainEpoch = -13
+	UpgradeTurboHeight          abi.ChainEpoch = -14
+	UpgradeHyperdriveHeight     abi.ChainEpoch = -15
+	UpgradeChocolateHeight      abi.ChainEpoch = -16
+	UpgradeOhSnapHeight         abi.ChainEpoch = -17
+	UpgradeSkyrHeight           abi.ChainEpoch = -18
+	UpgradeSharkHeight          abi.ChainEpoch = -19
+	UpgradeHyggeHeight          abi.ChainEpoch = -20
+	UpgradeLightningHeight      abi.ChainEpoch = -21
+	UpgradeThunderHeight        abi.ChainEpoch = -22
+	UpgradeWatermelonHeight     abi.ChainEpoch = -23
+	UpgradeWatermelonFixHeight  abi.ChainEpoch = -24
+	UpgradeWatermelonFix2Height abi.ChainEpoch = -25
 
 	DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 		0: DrandMainnet,
@@ -139,7 +141,3 @@ const BootstrapPeerThreshold = 1
 // ChainId defines the chain ID used in the Ethereum JSON-RPC endpoint.
 // As per https://github.com/ethereum-lists/chains
 const Eip155ChainId = 31415926
-
-// Reducing the delivery delay for equivocation of
-// consistent broadcast to just half a second.
-var CBDeliveryDelay = 500 * time.Millisecond

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.23.3"
+const BuildVersion = "1.25.1"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/builtin/builtin.go

@@ -23,6 +23,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 )
 
+var InitActorAddr = builtin.InitActorAddr
 var SystemActorAddr = builtin.SystemActorAddr
 var BurntFundsActorAddr = builtin.BurntFundsActorAddr
 var CronActorAddr = builtin.CronActorAddr

chain/actors/builtin/builtin.go.template

@@ -23,6 +23,7 @@ import (
 	smoothingtypes "github.com/filecoin-project/go-state-types/builtin/v8/util/smoothing"
 )
 
+var InitActorAddr = builtin.InitActorAddr
 var SystemActorAddr = builtin.SystemActorAddr
 var BurntFundsActorAddr = builtin.BurntFundsActorAddr
 var CronActorAddr = builtin.CronActorAddr

chain/actors/manifest.go

@@ -51,6 +51,12 @@ func RegisterManifest(av actorstypes.Version, manifestCid cid.Cid, entries map[s
 	}
 }
 
+func AddActorMeta(name string, codeId cid.Cid, av actorstypes.Version) {
+	manifestMx.Lock()
+	defer manifestMx.Unlock()
+	actorMeta[codeId] = actorEntry{name: name, version: av}
+}
+
 // GetManifest gets a loaded manifest.
 func GetManifest(av actorstypes.Version) (cid.Cid, bool) {
 	manifestMx.RLock()

chain/badtscache.go

@@ -3,14 +3,14 @@ package chain
 import (
 	"fmt"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/hashicorp/golang-lru/arc/v2"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/lotus/build"
 )
 
 type BadBlockCache struct {
-	badBlocks *lru.ARCCache[cid.Cid, BadBlockReason]
+	badBlocks *arc.ARCCache[cid.Cid, BadBlockReason]
 }
 
 type BadBlockReason struct {
@@ -43,7 +43,7 @@ func (bbr BadBlockReason) String() string {
 }
 
 func NewBadBlockCache() *BadBlockCache {
-	cache, err := lru.NewARC[cid.Cid, BadBlockReason](build.BadBlockCacheSize)
+	cache, err := arc.NewARC[cid.Cid, BadBlockReason](build.BadBlockCacheSize)
 	if err != nil {
 		panic(err) // ok
 	}

chain/beacon/drand/drand.go

@@ -235,3 +235,16 @@ func (db *DrandBeacon) maxBeaconRoundV2(latestTs uint64) uint64 {
 }
 
 var _ beacon.RandomBeacon = (*DrandBeacon)(nil)
+
+func BeaconScheduleFromDrandSchedule(dcs dtypes.DrandSchedule, genesisTime uint64, ps *pubsub.PubSub) (beacon.Schedule, error) {
+	shd := beacon.Schedule{}
+	for _, dc := range dcs {
+		bc, err := NewDrandBeacon(genesisTime, build.BlockDelaySecs, ps, dc.Config)
+		if err != nil {
+			return nil, xerrors.Errorf("creating drand beacon: %w", err)
+		}
+		shd = append(shd, beacon.BeaconPoint{Start: dc.Start, Beacon: bc})
+	}
+
+	return shd, nil
+}

chain/consensus/filcns/upgrades.go

@@ -1,6 +1,7 @@
 package filcns
 
 import (
+	"bytes"
 	"context"
 	_ "embed"
 	"fmt"
@@ -19,7 +20,9 @@ import (
 	actorstypes "github.com/filecoin-project/go-state-types/actors"
 	"github.com/filecoin-project/go-state-types/big"
 	nv18 "github.com/filecoin-project/go-state-types/builtin/v10/migration"
+	init11 "github.com/filecoin-project/go-state-types/builtin/v11/init"
 	nv19 "github.com/filecoin-project/go-state-types/builtin/v11/migration"
+	system11 "github.com/filecoin-project/go-state-types/builtin/v11/system"
 	nv21 "github.com/filecoin-project/go-state-types/builtin/v12/migration"
 	nv17 "github.com/filecoin-project/go-state-types/builtin/v9/migration"
 	"github.com/filecoin-project/go-state-types/manifest"
@@ -268,12 +271,21 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
 		Migration: UpgradeActorsV12,
 		PreMigrations: []stmgr.PreMigration{{
 			PreMigration:    PreUpgradeActorsV12,
-			StartWithin:     120,
+			StartWithin:     180,
 			DontStartWithin: 15,
 			StopWithin:      10,
 		}},
 		Expensive: true,
+	}, {
+		Height:    build.UpgradeWatermelonFixHeight,
+		Network:   network.Version21,
+		Migration: buildUpgradeActorsV12MinerFix(calibnetv12BuggyMinerCID1, calibnetv12BuggyManifestCID2),
 	},
+		{
+			Height:    build.UpgradeWatermelonFix2Height,
+			Network:   network.Version21,
+			Migration: buildUpgradeActorsV12MinerFix(calibnetv12BuggyMinerCID2, calibnetv12CorrectManifestCID1),
+		},
 	}
 
 	for _, u := range updates {
@@ -1869,6 +1881,18 @@ func UpgradeActorsV12(ctx context.Context, sm *stmgr.StateManager, cache stmgr.M
 	return newRoot, nil
 }
 
+var (
+	calibnetv12BuggyMinerCID1 = cid.MustParse("bafk2bzacecnh2ouohmonvebq7uughh4h3ppmg4cjsk74dzxlbbtlcij4xbzxq")
+	calibnetv12BuggyMinerCID2 = cid.MustParse("bafk2bzaced7emkbbnrewv5uvrokxpf5tlm4jslu2jsv77ofw2yqdglg657uie")
+
+	calibnetv12BuggyBundleSuffix1 = "calibrationnet-12-rc1"
+	calibnetv12BuggyBundleSuffix2 = "calibrationnet-12-rc2"
+
+	calibnetv12BuggyManifestCID1   = cid.MustParse("bafy2bzacedrunxfqta5skb7q7x32lnp4efz2oq7fn226ffm7fu5iqs62jkmvs")
+	calibnetv12BuggyManifestCID2   = cid.MustParse("bafy2bzacebl4w5ptfvuw6746w7ev562idkbf5ppq72e6zub22435ws2rukzru")
+	calibnetv12CorrectManifestCID1 = cid.MustParse("bafy2bzacednzb3pkrfnbfhmoqtb3bc6dgvxszpqklf3qcc7qzcage4ewzxsca")
+)
+
 func upgradeActorsV12Common(
 	ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache,
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
@@ -1894,13 +1918,50 @@ func upgradeActorsV12Common(
 		)
 	}
 
-	manifest, ok := actors.GetManifest(actorstypes.Version12)
-	if !ok {
-		return cid.Undef, xerrors.Errorf("no manifest CID for v12 upgrade")
+	// check whether or not this is a calibnet upgrade
+	// we do this because calibnet upgraded to a "wrong" actors bundle, which was then corrected
+	// we thus upgrade to calibrationnet-buggy in this upgrade
+	actorsIn, err := state.LoadStateTree(adtStore, root)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("loading state tree: %w", err)
+	}
+
+	initActor, err := actorsIn.GetActor(builtin.InitActorAddr)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor: %w", err)
+	}
+
+	var initState init11.State
+	if err := adtStore.Get(ctx, initActor.Head, &initState); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get system actor state: %w", err)
+	}
+
+	var manifestCid cid.Cid
+	if initState.NetworkName == "calibrationnet" {
+		embedded, ok := build.GetEmbeddedBuiltinActorsBundle(actorstypes.Version12, calibnetv12BuggyBundleSuffix1)
+		if !ok {
+			return cid.Undef, xerrors.Errorf("didn't find buggy calibrationnet bundle")
+		}
+
+		var err error
+		manifestCid, err = bundle.LoadBundle(ctx, writeStore, bytes.NewReader(embedded))
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to load buggy calibnet bundle: %w", err)
+		}
+
+		if manifestCid != calibnetv12BuggyManifestCID1 {
+			return cid.Undef, xerrors.Errorf("didn't find expected buggy calibnet bundle manifest: %s != %s", manifestCid, calibnetv12BuggyManifestCID1)
+		}
+	} else {
+		ok := false
+		manifestCid, ok = actors.GetManifest(actorstypes.Version12)
+		if !ok {
+			return cid.Undef, xerrors.Errorf("no manifest CID for v12 upgrade")
+		}
 	}
 
 	// Perform the migration
-	newHamtRoot, err := nv21.MigrateStateTree(ctx, adtStore, manifest, stateRoot.Actors, epoch, config,
+	newHamtRoot, err := nv21.MigrateStateTree(ctx, adtStore, manifestCid, stateRoot.Actors, epoch, config,
 		migrationLogger{}, cache)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("upgrading to actors v12: %w", err)
@@ -1928,6 +1989,165 @@ func upgradeActorsV12Common(
 	return newRoot, nil
 }
 
+// ////////////////////
+func buildUpgradeActorsV12MinerFix(oldBuggyMinerCID, newManifestCID cid.Cid) func(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
+	return func(ctx context.Context, sm *stmgr.StateManager, cache stmgr.MigrationCache, cb stmgr.ExecMonitor, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
+		stateStore := sm.ChainStore().StateBlockstore()
+		adtStore := store.ActorStore(ctx, stateStore)
+
+		// ensure that the manifest is loaded in the blockstore
+
+		// this loads the "correct" bundle for UpgradeWatermelonFix2Height
+		if err := bundle.LoadBundles(ctx, stateStore, actorstypes.Version12); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to load manifest bundle: %w", err)
+		}
+
+		// this loads the second buggy bundle, for UpgradeWatermelonFixHeight
+		embedded, ok := build.GetEmbeddedBuiltinActorsBundle(actorstypes.Version12, calibnetv12BuggyBundleSuffix2)
+		if !ok {
+			return cid.Undef, xerrors.Errorf("didn't find buggy calibrationnet bundle")
+		}
+
+		_, err := bundle.LoadBundle(ctx, stateStore, bytes.NewReader(embedded))
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to load buggy calibnet bundle: %w", err)
+		}
+
+		// now confirm we have the one we're migrating to
+		if haveManifest, err := stateStore.Has(ctx, newManifestCID); err != nil {
+			return cid.Undef, xerrors.Errorf("blockstore error when loading manifest %s: %w", newManifestCID, err)
+		} else if !haveManifest {
+			return cid.Undef, xerrors.Errorf("missing new manifest %s in blockstore", newManifestCID)
+		}
+
+		// Load input state tree
+		actorsIn, err := state.LoadStateTree(adtStore, root)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("loading state tree: %w", err)
+		}
+
+		// load old manifest data
+		systemActor, err := actorsIn.GetActor(builtin.SystemActorAddr)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to get system actor: %w", err)
+		}
+
+		var systemState system11.State
+		if err := adtStore.Get(ctx, systemActor.Head, &systemState); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to get system actor state: %w", err)
+		}
+
+		var oldManifestData manifest.ManifestData
+		if err := adtStore.Get(ctx, systemState.BuiltinActors, &oldManifestData); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to get old manifest data: %w", err)
+		}
+
+		// load new manifest
+		var newManifest manifest.Manifest
+		if err := adtStore.Get(ctx, newManifestCID, &newManifest); err != nil {
+			return cid.Undef, xerrors.Errorf("error reading actor manifest: %w", err)
+		}
+
+		if err := newManifest.Load(ctx, adtStore); err != nil {
+			return cid.Undef, xerrors.Errorf("error loading actor manifest: %w", err)
+		}
+
+		// build the CID mapping
+		codeMapping := make(map[cid.Cid]cid.Cid, len(oldManifestData.Entries))
+		for _, oldEntry := range oldManifestData.Entries {
+			newCID, ok := newManifest.Get(oldEntry.Name)
+			if !ok {
+				return cid.Undef, xerrors.Errorf("missing manifest entry for %s", oldEntry.Name)
+			}
+
+			// Note: we expect newCID to be the same as oldEntry.Code for all actors except the miner actor
+			codeMapping[oldEntry.Code] = newCID
+		}
+
+		// Create empty actorsOut
+
+		actorsOut, err := state.NewStateTree(adtStore, actorsIn.Version())
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to create new tree: %w", err)
+		}
+
+		// Perform the migration
+		err = actorsIn.ForEach(func(a address.Address, actor *types.Actor) error {
+			newCid, ok := codeMapping[actor.Code]
+			if !ok {
+				return xerrors.Errorf("didn't find mapping for %s", actor.Code)
+			}
+
+			return actorsOut.SetActor(a, &types.ActorV5{
+				Code:    newCid,
+				Head:    actor.Head,
+				Nonce:   actor.Nonce,
+				Balance: actor.Balance,
+				Address: actor.Address,
+			})
+		})
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to perform migration: %w", err)
+		}
+
+		systemState.BuiltinActors = newManifest.Data
+		newSystemHead, err := adtStore.Put(ctx, &systemState)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to put new system state: %w", err)
+		}
+
+		systemActor.Head = newSystemHead
+		if err = actorsOut.SetActor(builtin.SystemActorAddr, systemActor); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to put new system actor: %w", err)
+		}
+
+		// Sanity checking
+
+		err = actorsIn.ForEach(func(a address.Address, inActor *types.Actor) error {
+			outActor, err := actorsOut.GetActor(a)
+			if err != nil {
+				return xerrors.Errorf("failed to get actor in outTree: %w", err)
+			}
+
+			if inActor.Nonce != outActor.Nonce {
+				return xerrors.Errorf("mismatched nonce for actor %s", a)
+			}
+
+			if !inActor.Balance.Equals(outActor.Balance) {
+				return xerrors.Errorf("mismatched balance for actor %s: %d != %d", a, inActor.Balance, outActor.Balance)
+			}
+
+			if inActor.Address != outActor.Address && inActor.Address.String() != outActor.Address.String() {
+				return xerrors.Errorf("mismatched address for actor %s: %s != %s", a, inActor.Address, outActor.Address)
+			}
+
+			if inActor.Head != outActor.Head && a != builtin.SystemActorAddr {
+				return xerrors.Errorf("mismatched head for actor %s", a)
+			}
+
+			// Actor Codes are only expected to change for the miner actor
+			if inActor.Code != oldBuggyMinerCID && inActor.Code != outActor.Code {
+				return xerrors.Errorf("unexpected change in code for actor %s", a)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to sanity check migration: %w", err)
+		}
+
+		// Persist the result.
+		newRoot, err := actorsOut.Flush(ctx)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err)
+		}
+
+		return newRoot, nil
+	}
+}
+
+////////////////////
+
 // Example upgrade function if upgrade requires only code changes
 //func UpgradeActorsV9(ctx context.Context, sm *stmgr.StateManager, _ stmgr.MigrationCache, _ stmgr.ExecMonitor, root cid.Cid, _ abi.ChainEpoch, _ *types.TipSet) (cid.Cid, error) {
 //	buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), blockstore.NewMemorySync())

chain/events/filter/event.go

@@ -388,7 +388,7 @@ func (m *EventFilterManager) Install(ctx context.Context, minHeight, maxHeight a
 
 	if m.EventIndex != nil && minHeight != -1 && minHeight < currentHeight {
 		// Filter needs historic events
-		if err := m.EventIndex.PrefillFilter(ctx, f); err != nil {
+		if err := m.EventIndex.PrefillFilter(ctx, f, true); err != nil {
 			return nil, err
 		}
 	}

chain/events/filter/index.go

@@ -46,6 +46,7 @@ var ddls = []string{
 	)`,
 
 	`CREATE INDEX IF NOT EXISTS height_tipset_key ON event (height,tipset_key)`,
+	`CREATE INDEX IF NOT EXISTS event_emitter_addr ON event (emitter_addr)`,
 
 	`CREATE TABLE IF NOT EXISTS event_entry (
 		event_id INTEGER,
@@ -56,6 +57,8 @@ var ddls = []string{
 		value BLOB NOT NULL
 	)`,
 
+	`CREATE INDEX IF NOT EXISTS event_entry_key_index ON event_entry (key)`,
+
 	// metadata containing version of schema
 	`CREATE TABLE IF NOT EXISTS _meta (
     	version UINT64 NOT NULL UNIQUE
@@ -63,6 +66,7 @@ var ddls = []string{
 
 	`INSERT OR IGNORE INTO _meta (version) VALUES (1)`,
 	`INSERT OR IGNORE INTO _meta (version) VALUES (2)`,
+	`INSERT OR IGNORE INTO _meta (version) VALUES (3)`,
 }
 
 var (
@@ -70,7 +74,7 @@ var (
 )
 
 const (
-	schemaVersion = 2
+	schemaVersion = 3
 
 	eventExists          = `SELECT MAX(id) FROM event WHERE height=? AND tipset_key=? AND tipset_key_cid=? AND emitter_addr=? AND event_index=? AND message_cid=? AND message_index=?`
 	insertEvent          = `INSERT OR IGNORE INTO event(height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`
@@ -321,6 +325,22 @@ func NewEventIndex(ctx context.Context, path string, chainStore *store.ChainStor
 			version = 2
 		}
 
+		if version == 2 {
+			log.Infof("upgrading event index from version 1 to version 2")
+
+			// to upgrade to version 3 we only need to create an index on the event_entry.key column
+			// and on the event.emitter_addr column
+			// which means we can just reapply the schema (it will not have any effect on existing data)
+			for _, ddl := range ddls {
+				if _, err := db.Exec(ddl); err != nil {
+					_ = db.Close()
+					return nil, xerrors.Errorf("could not upgrade index to version 3, exec ddl %q: %w", ddl, err)
+				}
+			}
+
+			version = 3
+		}
+
 		if version != schemaVersion {
 			_ = db.Close()
 			return nil, xerrors.Errorf("invalid database version: got %d, expected %d", version, schemaVersion)
@@ -481,7 +501,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever
 }
 
 // PrefillFilter fills a filter's collection of events from the historic index
-func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter) error {
+func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter, excludeReverted bool) error {
 	clauses := []string{}
 	values := []any{}
 	joins := []string{}
@@ -500,6 +520,11 @@ func (ei *EventIndex) PrefillFilter(ctx context.Context, f *EventFilter) error {
 		}
 	}
 
+	if excludeReverted {
+		clauses = append(clauses, "event.reverted=?")
+		values = append(values, false)
+	}
+
 	if len(f.addresses) > 0 {
 		subclauses := []string{}
 		for _, addr := range f.addresses {

chain/events/filter/index_test.go

@@ -272,7 +272,7 @@ func TestEventIndexPrefillFilter(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc // appease lint
 		t.Run(tc.name, func(t *testing.T) {
-			if err := ei.PrefillFilter(context.Background(), tc.filter); err != nil {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, false); err != nil {
 				require.NoError(t, err, "prefill filter events")
 			}
 
@@ -281,3 +281,619 @@ func TestEventIndexPrefillFilter(t *testing.T) {
 		})
 	}
 }
+
+func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) {
+	rng := pseudo.New(pseudo.NewSource(299792458))
+	a1 := randomF4Addr(t, rng)
+	a2 := randomF4Addr(t, rng)
+	a3 := randomF4Addr(t, rng)
+
+	a1ID := abi.ActorID(1)
+	a2ID := abi.ActorID(2)
+
+	addrMap := addressMap{}
+	addrMap.add(a1ID, a1)
+	addrMap.add(a2ID, a2)
+
+	ev1 := fakeEvent(
+		a1ID,
+		[]kv{
+			{k: "type", v: []byte("approval")},
+			{k: "signer", v: []byte("addr1")},
+		},
+		[]kv{
+			{k: "amount", v: []byte("2988181")},
+		},
+	)
+	ev2 := fakeEvent(
+		a2ID,
+		[]kv{
+			{k: "type", v: []byte("approval")},
+			{k: "signer", v: []byte("addr2")},
+		},
+		[]kv{
+			{k: "amount", v: []byte("2988182")},
+		},
+	)
+
+	st := newStore()
+	events := []*types.Event{ev1}
+	revertedEvents := []*types.Event{ev2}
+	em := executedMessage{
+		msg: fakeMessage(randomF4Addr(t, rng), randomF4Addr(t, rng)),
+		rct: fakeReceipt(t, rng, st, events),
+		evs: events,
+	}
+	revertedEm := executedMessage{
+		msg: fakeMessage(randomF4Addr(t, rng), randomF4Addr(t, rng)),
+		rct: fakeReceipt(t, rng, st, revertedEvents),
+		evs: revertedEvents,
+	}
+
+	events14000 := buildTipSetEvents(t, rng, 14000, em)
+	revertedEvents14000 := buildTipSetEvents(t, rng, 14000, revertedEm)
+	cid14000, err := events14000.msgTs.Key().Cid()
+	require.NoError(t, err, "tipset cid")
+	reveredCID14000, err := revertedEvents14000.msgTs.Key().Cid()
+	require.NoError(t, err, "tipset cid")
+
+	noCollectedEvents := []*CollectedEvent{}
+	oneCollectedEvent := []*CollectedEvent{
+		{
+			Entries:     ev1.Entries,
+			EmitterAddr: a1,
+			EventIdx:    0,
+			Reverted:    false,
+			Height:      14000,
+			TipSetKey:   events14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      em.msg.Cid(),
+		},
+	}
+	twoCollectedEvent := []*CollectedEvent{
+		{
+			Entries:     ev1.Entries,
+			EmitterAddr: a1,
+			EventIdx:    0,
+			Reverted:    false,
+			Height:      14000,
+			TipSetKey:   events14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      em.msg.Cid(),
+		},
+		{
+			Entries:     ev2.Entries,
+			EmitterAddr: a2,
+			EventIdx:    0,
+			Reverted:    true,
+			Height:      14000,
+			TipSetKey:   revertedEvents14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      revertedEm.msg.Cid(),
+		},
+	}
+	oneCollectedRevertedEvent := []*CollectedEvent{
+		{
+			Entries:     ev2.Entries,
+			EmitterAddr: a2,
+			EventIdx:    0,
+			Reverted:    true,
+			Height:      14000,
+			TipSetKey:   revertedEvents14000.msgTs.Key(),
+			MsgIdx:      0,
+			MsgCid:      revertedEm.msg.Cid(),
+		},
+	}
+
+	workDir, err := os.MkdirTemp("", "lotusevents")
+	require.NoError(t, err, "create temporary work directory")
+
+	defer func() {
+		_ = os.RemoveAll(workDir)
+	}()
+	t.Logf("using work dir %q", workDir)
+
+	dbPath := filepath.Join(workDir, "actorevents.db")
+
+	ei, err := NewEventIndex(context.Background(), dbPath, nil)
+	require.NoError(t, err, "create event index")
+	if err := ei.CollectEvents(context.Background(), revertedEvents14000, false, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "collect reverted events")
+	}
+	if err := ei.CollectEvents(context.Background(), revertedEvents14000, true, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "revert reverted events")
+	}
+	if err := ei.CollectEvents(context.Background(), events14000, false, addrMap.ResolveAddress); err != nil {
+		require.NoError(t, err, "collect events")
+	}
+
+	inclusiveTestCases := []struct {
+		name   string
+		filter *EventFilter
+		te     *TipSetEvents
+		want   []*CollectedEvent
+	}{
+		{
+			name: "nomatch tipset min height",
+			filter: &EventFilter{
+				minHeight: 14001,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch tipset max height",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: 13999,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match tipset min height",
+			filter: &EventFilter{
+				minHeight: 14000,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: cid14000,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: reveredCID14000,
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "nomatch address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a3},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match address 2",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a2},
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "match address 1",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a1},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "match one entry with alternate values",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: twoCollectedEvent,
+		},
+		{
+			name: "nomatch one entry by missing value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry by missing key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"method": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr2"),
+					},
+				},
+			},
+			te:   revertedEvents14000,
+			want: oneCollectedRevertedEvent,
+		},
+		{
+			name: "nomatch one entry with one mismatching key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"approver": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one mismatching value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr3"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988181"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988182"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+	}
+
+	exclusiveTestCases := []struct {
+		name   string
+		filter *EventFilter
+		te     *TipSetEvents
+		want   []*CollectedEvent
+	}{
+		{
+			name: "nomatch tipset min height",
+			filter: &EventFilter{
+				minHeight: 14001,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch tipset max height",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: 13999,
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match tipset min height",
+			filter: &EventFilter{
+				minHeight: 14000,
+				maxHeight: -1,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: cid14000,
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match tipset cid but reverted",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				tipsetCid: reveredCID14000,
+			},
+			te:   revertedEvents14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a3},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch address 2 but reverted",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a2},
+			},
+			te:   revertedEvents14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match address",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				addresses: []address.Address{a1},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "match one entry with alternate values",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "nomatch one entry by missing value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("cancel"),
+						[]byte("propose"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry by missing key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"method": {
+						[]byte("approval"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "match one entry with multiple keys",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: oneCollectedEvent,
+		},
+		{
+			name: "nomatch one entry with one mismatching key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"approver": {
+						[]byte("addr1"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with matching reverted value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr2"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one mismatching value",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"type": {
+						[]byte("approval"),
+					},
+					"signer": {
+						[]byte("addr3"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+		{
+			name: "nomatch one entry with one unindexed key",
+			filter: &EventFilter{
+				minHeight: -1,
+				maxHeight: -1,
+				keys: map[string][][]byte{
+					"amount": {
+						[]byte("2988181"),
+					},
+				},
+			},
+			te:   events14000,
+			want: noCollectedEvents,
+		},
+	}
+
+	for _, tc := range inclusiveTestCases {
+		tc := tc // appease lint
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, false); err != nil {
+				require.NoError(t, err, "prefill filter events")
+			}
+
+			coll := tc.filter.TakeCollectedEvents(context.Background())
+			require.ElementsMatch(t, coll, tc.want, tc.name)
+		})
+	}
+
+	for _, tc := range exclusiveTestCases {
+		tc := tc // appease lint
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ei.PrefillFilter(context.Background(), tc.filter, true); err != nil {
+				require.NoError(t, err, "prefill filter events")
+			}
+
+			coll := tc.filter.TakeCollectedEvents(context.Background())
+			require.ElementsMatch(t, coll, tc.want, tc.name)
+		})
+	}
+}

chain/events/message_cache.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"sync"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/hashicorp/golang-lru/arc/v2"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/lotus/api"
@@ -14,11 +14,11 @@ type messageCache struct {
 	api EventAPI
 
 	blockMsgLk    sync.Mutex
-	blockMsgCache *lru.ARCCache[cid.Cid, *api.BlockMessages]
+	blockMsgCache *arc.ARCCache[cid.Cid, *api.BlockMessages]
 }
 
 func newMessageCache(a EventAPI) *messageCache {
-	blsMsgCache, _ := lru.NewARC[cid.Cid, *api.BlockMessages](500)
+	blsMsgCache, _ := arc.NewARC[cid.Cid, *api.BlockMessages](500)
 
 	return &messageCache{
 		api:           a,

chain/exchange/cbor_gen.go

@@ -306,9 +306,9 @@ func (t *Response) UnmarshalCBOR(r io.Reader) (err error) {
 	return nil
 }
 
-var lengthBufCompactedMessages = []byte{132}
+var lengthBufCompactedMessagesCBOR = []byte{132}
 
-func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
@@ -316,12 +316,12 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write(lengthBufCompactedMessages); err != nil {
+	if _, err := cw.Write(lengthBufCompactedMessagesCBOR); err != nil {
 		return err
 	}
 
 	// t.Bls ([]*types.Message) (slice)
-	if len(t.Bls) > cbg.MaxLength {
+	if len(t.Bls) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Bls was too long")
 	}
 
@@ -334,7 +334,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 	if len(t.BlsIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
 	}
@@ -343,24 +343,13 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.BlsIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 
 	// t.Secpk ([]*types.SignedMessage) (slice)
-	if len(t.Secpk) > cbg.MaxLength {
+	if len(t.Secpk) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Secpk was too long")
 	}
 
@@ -373,7 +362,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 	if len(t.SecpkIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
 	}
@@ -382,26 +371,15 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.SecpkIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 	return nil
 }
 
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = CompactedMessages{}
+func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = CompactedMessagesCBOR{}
 
 	cr := cbg.NewCborReader(r)
 
@@ -430,7 +408,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Bls: array too large (%d)", extra)
 	}
 
@@ -471,7 +449,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -487,7 +465,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.BlsIncludes = make([][]uint64, extra)
+		t.BlsIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -499,47 +477,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.BlsIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.BlsIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.BlsIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.BlsIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 
@@ -550,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Secpk: array too large (%d)", extra)
 	}
 
@@ -591,7 +535,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -607,7 +551,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.SecpkIncludes = make([][]uint64, extra)
+		t.SecpkIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -619,47 +563,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.SecpkIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.SecpkIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.SecpkIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.SecpkIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 

chain/exchange/client.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"time"
 
@@ -23,6 +24,10 @@ import (
 	"github.com/filecoin-project/lotus/lib/peermgr"
 )
 
+// Set the max exchange message size to 120MiB. Purely based on gas numbers, we can include ~8MiB of
+// messages per block, so I've set this to 120MiB to be _very_ safe.
+const maxExchangeMessageSize = (15 * 8) << 20
+
 // client implements exchange.Client, using the libp2p ChainExchange protocol
 // as the fetching mechanism.
 type client struct {
@@ -434,10 +439,11 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 		log.Warnw("CloseWrite err", "error", err)
 	}
 
-	// Read response.
+	// Read response, limiting the size of the response to maxExchangeMessageSize as we allow a
+	// lot of messages (10k+) but they'll mostly be quite small.
 	var res Response
 	err = cborutil.ReadCborRPC(
-		bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)),
+		bufio.NewReader(io.LimitReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline), maxExchangeMessageSize)),
 		&res)
 	if err != nil {
 		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)

chain/exchange/protocol.go

@@ -154,6 +154,8 @@ type BSTipSet struct {
 // FIXME: The logic to decompress this structure should belong
 //
 //	to itself, not to the consumer.
+//
+// NOTE: Max messages is: BlockMessageLimit (10k) * MaxTipsetSize (15) = 150k
 type CompactedMessages struct {
 	Bls         []*types.Message
 	BlsIncludes [][]uint64

chain/exchange/protocol_encoding.go

@@ -0,0 +1,125 @@
+package exchange
+
+import (
+	"fmt"
+	"io"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/build"
+	types "github.com/filecoin-project/lotus/chain/types"
+)
+
+// Type used for encoding/decoding compacted messages. This is a ustom type as we need custom limits.
+// - Max messages is 150,000 as that's 15 times the max block size (in messages). It needs to be
+// large enough to cover a full tipset full of full blocks.
+type CompactedMessagesCBOR struct {
+	Bls         []*types.Message `cborgen:"maxlen=150000"`
+	BlsIncludes []messageIndices
+
+	Secpk         []*types.SignedMessage `cborgen:"maxlen=150000"`
+	SecpkIncludes []messageIndices
+}
+
+// Unmarshal into the "decoding" struct, then copy into the actual struct.
+func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
+	var c CompactedMessagesCBOR
+	if err := c.UnmarshalCBOR(r); err != nil {
+		return err
+	}
+	t.Bls = c.Bls
+	t.BlsIncludes = make([][]uint64, len(c.BlsIncludes))
+	for i, v := range c.BlsIncludes {
+		t.BlsIncludes[i] = v.v
+	}
+	t.Secpk = c.Secpk
+	t.SecpkIncludes = make([][]uint64, len(c.SecpkIncludes))
+	for i, v := range c.SecpkIncludes {
+		t.SecpkIncludes[i] = v.v
+	}
+	return nil
+}
+
+// Copy into the encoding struct, then marshal.
+func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	var c CompactedMessagesCBOR
+	c.Bls = t.Bls
+	c.BlsIncludes = make([]messageIndices, len(t.BlsIncludes))
+	for i, v := range t.BlsIncludes {
+		c.BlsIncludes[i].v = v
+	}
+	c.Secpk = t.Secpk
+	c.SecpkIncludes = make([]messageIndices, len(t.SecpkIncludes))
+	for i, v := range t.SecpkIncludes {
+		c.SecpkIncludes[i].v = v
+	}
+	return c.MarshalCBOR(w)
+}
+
+// this needs to be a struct or cborgen will peak into it and ignore the Unmarshal/Marshal functions
+type messageIndices struct {
+	v []uint64
+}
+
+func (t *messageIndices) UnmarshalCBOR(r io.Reader) (err error) {
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra > uint64(build.BlockMessageLimit) {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	if extra > 0 {
+		t.v = make([]uint64, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+		maj, extra, err := cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.v[i] = extra
+
+	}
+	return nil
+}
+
+func (t *messageIndices) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if len(t.v) > build.BlockMessageLimit {
+		return xerrors.Errorf("Slice value in field v was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.v))); err != nil {
+		return err
+	}
+	for _, v := range t.v {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, v); err != nil {
+			return err
+		}
+	}
+	return nil
+}

chain/gen/gen.go

@@ -362,7 +362,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add
 		rbase = entries[len(entries)-1]
 	}
 
-	eproof, err := IsRoundWinner(ctx, pts, round, m, rbase, mbi, mc)
+	eproof, err := IsRoundWinner(ctx, round, m, rbase, mbi, mc)
 	if err != nil {
 		return nil, nil, nil, xerrors.Errorf("checking round winner failed: %w", err)
 	}
@@ -449,18 +449,19 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
 }
 
 func (cg *ChainGen) NextTipSetFromMinersWithMessagesAndNulls(base *types.TipSet, miners []address.Address, msgs [][]*types.SignedMessage, nulls abi.ChainEpoch) (*store.FullTipSet, error) {
+	ctx := context.TODO()
 	var blks []*types.FullBlock
 
 	for round := base.Height() + nulls + 1; len(blks) == 0; round++ {
 		for mi, m := range miners {
-			bvals, et, ticket, err := cg.nextBlockProof(context.TODO(), base, m, round)
+			bvals, et, ticket, err := cg.nextBlockProof(ctx, base, m, round)
 			if err != nil {
 				return nil, xerrors.Errorf("next block proof: %w", err)
 			}
 
 			if et != nil {
 				// TODO: maybe think about passing in more real parameters to this?
-				wpost, err := cg.eppProvs[m].ComputeProof(context.TODO(), nil, nil, round, network.Version0)
+				wpost, err := cg.eppProvs[m].ComputeProof(ctx, nil, nil, round, network.Version0)
 				if err != nil {
 					return nil, err
 				}
@@ -476,8 +477,18 @@ func (cg *ChainGen) NextTipSetFromMinersWithMessagesAndNulls(base *types.TipSet,
 	}
 
 	fts := store.NewFullTipSet(blks)
-	if err := cg.cs.PutTipSet(context.TODO(), fts.TipSet()); err != nil {
-		return nil, err
+	if err := cg.cs.PersistTipsets(ctx, []*types.TipSet{fts.TipSet()}); err != nil {
+		return nil, xerrors.Errorf("failed to persist tipset: %w", err)
+	}
+
+	for _, blk := range blks {
+		if err := cg.cs.AddToTipSetTracker(ctx, blk.Header); err != nil {
+			return nil, xerrors.Errorf("failed to add to tipset tracker: %w", err)
+		}
+	}
+
+	if err := cg.cs.RefreshHeaviestTipSet(ctx, fts.TipSet().Height()); err != nil {
+		return nil, xerrors.Errorf("failed to put tipset: %w", err)
 	}
 
 	cg.CurTipset = fts
@@ -628,7 +639,7 @@ func (wpp *wppProvider) ComputeProof(context.Context, []proof7.ExtendedSectorInf
 	return ValidWpostForTesting, nil
 }
 
-func IsRoundWinner(ctx context.Context, ts *types.TipSet, round abi.ChainEpoch,
+func IsRoundWinner(ctx context.Context, round abi.ChainEpoch,
 	miner address.Address, brand types.BeaconEntry, mbi *api.MiningBaseInfo, a MiningCheckAPI) (*types.ElectionProof, error) {
 
 	buf := new(bytes.Buffer)

chain/messagepool/messagepool.go

@@ -63,6 +63,9 @@ var MaxNonceGap = uint64(4)
 
 const MaxMessageSize = 64 << 10 // 64KiB
 
+// NOTE: When adding a new error type, please make sure to add the new error type in
+// func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message)
+// in /chain/sub/incoming.go
 var (
 	ErrMessageTooBig = errors.New("message too big")
 

chain/stmgr/forks.go

@@ -178,16 +178,18 @@ func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, heig
 	retCid := root
 	u := sm.stateMigrations[height]
 	if u != nil && u.upgrade != nil {
-		migCid, ok, err := u.migrationResultCache.Get(ctx, root)
-		if err == nil {
-			if ok {
-				log.Infow("CACHED migration", "height", height, "from", root, "to", migCid)
-				return migCid, nil
+		if height != build.UpgradeWatermelonFixHeight {
+			migCid, ok, err := u.migrationResultCache.Get(ctx, root)
+			if err == nil {
+				if ok {
+					log.Infow("CACHED migration", "height", height, "from", root, "to", migCid)
+					return migCid, nil
+				}
+			} else if !errors.Is(err, datastore.ErrNotFound) {
+				log.Errorw("failed to lookup previous migration result", "err", err)
+			} else {
+				log.Debug("no cached migration found, migrating from scratch")
 			}
-		} else if !errors.Is(err, datastore.ErrNotFound) {
-			log.Errorw("failed to lookup previous migration result", "err", err)
-		} else {
-			log.Debug("no cached migration found, migrating from scratch")
 		}
 
 		startTime := time.Now()
@@ -195,6 +197,7 @@ func (sm *StateManager) HandleStateForks(ctx context.Context, root cid.Cid, heig
 		// Yes, we clone the cache, even for the final upgrade epoch. Why? Reverts. We may
 		// have to migrate multiple times.
 		tmpCache := u.cache.Clone()
+		var err error
 		retCid, err = u.upgrade(ctx, sm, tmpCache, cb, root, height, ts)
 		if err != nil {
 			log.Errorw("FAILED migration", "height", height, "from", root, "error", err)

chain/stmgr/stmgr.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"sync"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/hashicorp/golang-lru/arc/v2"
 	"github.com/ipfs/go-cid"
 	dstore "github.com/ipfs/go-datastore"
 	cbor "github.com/ipfs/go-ipld-cbor"
@@ -156,7 +156,7 @@ type StateManager struct {
 
 	// We keep a small cache for calls to ExecutionTrace which helps improve
 	// performance for node operators like exchanges and block explorers
-	execTraceCache *lru.ARCCache[types.TipSetKey, tipSetCacheEntry]
+	execTraceCache *arc.ARCCache[types.TipSetKey, tipSetCacheEntry]
 	// We need a lock while making the copy as to prevent other callers
 	// overwrite the cache while making the copy
 	execTraceCacheLock sync.Mutex
@@ -213,10 +213,10 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
 	}
 
 	log.Debugf("execTraceCache size: %d", execTraceCacheSize)
-	var execTraceCache *lru.ARCCache[types.TipSetKey, tipSetCacheEntry]
+	var execTraceCache *arc.ARCCache[types.TipSetKey, tipSetCacheEntry]
 	var err error
 	if execTraceCacheSize > 0 {
-		execTraceCache, err = lru.NewARC[types.TipSetKey, tipSetCacheEntry](execTraceCacheSize)
+		execTraceCache, err = arc.NewARC[types.TipSetKey, tipSetCacheEntry](execTraceCacheSize)
 		if err != nil {
 			return nil, err
 		}

chain/stmgr/supply.go

@@ -388,6 +388,14 @@ func (sm *StateManager) GetCirculatingSupply(ctx context.Context, height abi.Cha
 	circ := big.Zero()
 	unCirc := big.Zero()
 	err := st.ForEach(func(a address.Address, actor *types.Actor) error {
+		// this can be a lengthy operation, we need to cancel early when
+		// the context is cancelled to avoid resource exhaustion
+		select {
+		case <-ctx.Done():
+			// this will cause ForEach to return
+			return ctx.Err()
+		default:
+		}
 		switch {
 		case actor.Balance.IsZero():
 			// Do nothing for zero-balance actors

chain/store/checkpoint_test.go

@@ -70,7 +70,7 @@ func TestChainCheckpoint(t *testing.T) {
 	}
 
 	// See if the chain will take the fork, it shouldn't.
-	err = cs.MaybeTakeHeavierTipSet(context.Background(), last)
+	err = cs.RefreshHeaviestTipSet(context.Background(), last.Height())
 	require.NoError(t, err)
 	head = cs.GetHeaviestTipSet()
 	require.True(t, head.Equals(checkpoint))
@@ -80,7 +80,7 @@ func TestChainCheckpoint(t *testing.T) {
 	require.NoError(t, err)
 
 	// Now switch to the other fork.
-	err = cs.MaybeTakeHeavierTipSet(context.Background(), last)
+	err = cs.RefreshHeaviestTipSet(context.Background(), last.Height())
 	require.NoError(t, err)
 	head = cs.GetHeaviestTipSet()
 	require.True(t, head.Equals(last))

chain/store/index_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/mock"
 )
 
@@ -47,28 +48,29 @@ func TestIndexSeeks(t *testing.T) {
 	}
 
 	cur := mock.TipSet(gen)
-	if err := cs.PutTipSet(ctx, mock.TipSet(gen)); err != nil {
-		t.Fatal(err)
-	}
+
 	assert.NoError(t, cs.SetGenesis(ctx, gen))
 
 	// Put 113 blocks from genesis
 	for i := 0; i < 113; i++ {
-		nextts := mock.TipSet(mock.MkBlock(cur, 1, 1))
-
-		if err := cs.PutTipSet(ctx, nextts); err != nil {
-			t.Fatal(err)
-		}
+		nextBlk := mock.MkBlock(cur, 1, 1)
+		nextts := mock.TipSet(nextBlk)
+		assert.NoError(t, cs.PersistTipsets(ctx, []*types.TipSet{nextts}))
+		assert.NoError(t, cs.AddToTipSetTracker(ctx, nextBlk))
 		cur = nextts
 	}
 
+	assert.NoError(t, cs.RefreshHeaviestTipSet(ctx, cur.Height()))
+
 	// Put 50 null epochs + 1 block
 	skip := mock.MkBlock(cur, 1, 1)
 	skip.Height += 50
-
 	skipts := mock.TipSet(skip)
 
-	if err := cs.PutTipSet(ctx, skipts); err != nil {
+	assert.NoError(t, cs.PersistTipsets(ctx, []*types.TipSet{skipts}))
+	assert.NoError(t, cs.AddToTipSetTracker(ctx, skip))
+
+	if err := cs.RefreshHeaviestTipSet(ctx, skip.Height); err != nil {
 		t.Fatal(err)
 	}
 

chain/store/snapshot.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ipld/go-car"
 	carutil "github.com/ipld/go-car/util"
 	carv2 "github.com/ipld/go-car/v2"
-	mh "github.com/multiformats/go-multihash"
+	"github.com/multiformats/go-multicodec"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.uber.org/atomic"
 	"golang.org/x/sync/errgroup"
@@ -369,14 +369,16 @@ func (s *walkScheduler) Wait() error {
 }
 
 func (s *walkScheduler) enqueueIfNew(task walkTask) {
-	if task.c.Prefix().MhType == mh.IDENTITY {
+	if multicodec.Code(task.c.Prefix().MhType) == multicodec.Identity {
 		//log.Infow("ignored", "cid", todo.c.String())
 		return
 	}
 
-	// This lets through RAW and CBOR blocks, the only two types that we
-	// end up writing to the exported CAR.
-	if task.c.Prefix().Codec != cid.Raw && task.c.Prefix().Codec != cid.DagCBOR {
+	// This lets through RAW, CBOR, and DagCBOR blocks, the only types that we end up writing to
+	// the exported CAR.
+	switch multicodec.Code(task.c.Prefix().Codec) {
+	case multicodec.Cbor, multicodec.DagCbor, multicodec.Raw:
+	default:
 		//log.Infow("ignored", "cid", todo.c.String())
 		return
 	}
@@ -450,7 +452,8 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 	// We exported the ipld block. If it wasn't a CBOR block, there's nothing
 	// else to do and we can bail out early as it won't have any links
 	// etc.
-	if t.c.Prefix().Codec != cid.DagCBOR || t.c.Prefix().MhType == mh.IDENTITY {
+	if multicodec.Code(t.c.Prefix().Codec) != multicodec.DagCbor ||
+		multicodec.Code(t.c.Prefix().MhType) == multicodec.Identity {
 		return nil
 	}
 
@@ -683,14 +686,13 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 				prefix := c.Prefix()
 
 				// Don't include identity CIDs.
-				if prefix.MhType == mh.IDENTITY {
+				if multicodec.Code(prefix.MhType) == multicodec.Identity {
 					continue
 				}
 
-				// We only include raw and dagcbor, for now.
-				// Raw for "code" CIDs.
-				switch prefix.Codec {
-				case cid.Raw, cid.DagCBOR:
+				// We only include raw, cbor, and dagcbor, for now.
+				switch multicodec.Code(prefix.Codec) {
+				case multicodec.Cbor, multicodec.DagCbor, multicodec.Raw:
 				default:
 					continue
 				}
@@ -722,7 +724,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 }
 
 func recurseLinks(ctx context.Context, bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
-	if root.Prefix().Codec != cid.DagCBOR {
+	if multicodec.Code(root.Prefix().Codec) != multicodec.DagCbor {
 		return in, nil
 	}
 

chain/store/store.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 	"time"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/hashicorp/golang-lru/arc/v2"
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	dstore "github.com/ipfs/go-datastore"
@@ -120,8 +120,8 @@ type ChainStore struct {
 	reorgCh        chan<- reorg
 	reorgNotifeeCh chan ReorgNotifee
 
-	mmCache *lru.ARCCache[cid.Cid, mmCids]
-	tsCache *lru.ARCCache[types.TipSetKey, *types.TipSet]
+	mmCache *arc.ARCCache[cid.Cid, mmCids]
+	tsCache *arc.ARCCache[types.TipSetKey, *types.TipSet]
 
 	evtTypes [1]journal.EventType
 	journal  journal.Journal
@@ -133,8 +133,8 @@ type ChainStore struct {
 }
 
 func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dstore.Batching, weight WeightFunc, j journal.Journal) *ChainStore {
-	c, _ := lru.NewARC[cid.Cid, mmCids](DefaultMsgMetaCacheSize)
-	tsc, _ := lru.NewARC[types.TipSetKey, *types.TipSet](DefaultTipSetCacheSize)
+	c, _ := arc.NewARC[cid.Cid, mmCids](DefaultMsgMetaCacheSize)
+	tsc, _ := arc.NewARC[types.TipSetKey, *types.TipSet](DefaultTipSetCacheSize)
 	if j == nil {
 		j = journal.NilJournal()
 	}
@@ -367,49 +367,32 @@ func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid)
 func (cs *ChainStore) SetGenesis(ctx context.Context, b *types.BlockHeader) error {
 	ts, err := types.NewTipSet([]*types.BlockHeader{b})
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to construct genesis tipset: %w", err)
 	}
 
-	if err := cs.PutTipSet(ctx, ts); err != nil {
-		return err
+	if err := cs.PersistTipsets(ctx, []*types.TipSet{ts}); err != nil {
+		return xerrors.Errorf("failed to persist genesis tipset: %w", err)
+	}
+
+	if err := cs.AddToTipSetTracker(ctx, b); err != nil {
+		return xerrors.Errorf("failed to add genesis tipset to tracker: %w", err)
+	}
+
+	if err := cs.RefreshHeaviestTipSet(ctx, ts.Height()); err != nil {
+		return xerrors.Errorf("failed to put genesis tipset: %w", err)
 	}
 
 	return cs.metadataDs.Put(ctx, dstore.NewKey("0"), b.Cid().Bytes())
 }
 
-func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
-	if err := cs.PersistTipsets(ctx, []*types.TipSet{ts}); err != nil {
-		return xerrors.Errorf("failed to persist tipset: %w", err)
-	}
-
-	expanded, err := cs.expandTipset(ctx, ts.Blocks()[0])
-	if err != nil {
-		return xerrors.Errorf("errored while expanding tipset: %w", err)
-	}
-
-	if expanded.Key() != ts.Key() {
-		log.Debugf("expanded %s into %s\n", ts.Cids(), expanded.Cids())
-
-		tsBlk, err := expanded.Key().ToStorageBlock()
-		if err != nil {
-			return xerrors.Errorf("failed to get tipset key block: %w", err)
-		}
-
-		if err = cs.chainLocalBlockstore.Put(ctx, tsBlk); err != nil {
-			return xerrors.Errorf("failed to put tipset key block: %w", err)
-		}
-	}
-
-	if err := cs.MaybeTakeHeavierTipSet(ctx, expanded); err != nil {
-		return xerrors.Errorf("MaybeTakeHeavierTipSet failed in PutTipSet: %w", err)
-	}
-	return nil
-}
-
-// MaybeTakeHeavierTipSet evaluates the incoming tipset and locks it in our
-// internal state as our new head, if and only if it is heavier than the current
-// head and does not exceed the maximum fork length.
-func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipSet) error {
+// RefreshHeaviestTipSet receives a newTsHeight at which a new tipset might exist. It then:
+// - "refreshes" the heaviest tipset that can be formed at its current heaviest height
+//   - if equivocation is detected among the miners of the current heaviest tipset, the head is immediately updated to the heaviest tipset that can be formed in a range of 5 epochs
+//
+// - forms the best tipset that can be formed at the _input_ height
+// - compares the three tipset weights: "current" heaviest tipset, "refreshed" tipset, and best tipset at newTsHeight
+// - updates "current" heaviest to the heaviest of those 3 tipsets (if an update is needed), assuming it doesn't violate the maximum fork rule
+func (cs *ChainStore) RefreshHeaviestTipSet(ctx context.Context, newTsHeight abi.ChainEpoch) error {
 	for {
 		cs.heaviestLk.Lock()
 		if len(cs.reorgCh) < reorgChBuf/2 {
@@ -426,39 +409,90 @@ func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipS
 
 	defer cs.heaviestLk.Unlock()
 
-	if ts.Equals(cs.heaviest) {
+	heaviestWeight, err := cs.weight(ctx, cs.StateBlockstore(), cs.heaviest)
+	if err != nil {
+		return xerrors.Errorf("failed to calculate currentHeaviest's weight: %w", err)
+	}
+
+	heaviestHeight := abi.ChainEpoch(0)
+	if cs.heaviest != nil {
+		heaviestHeight = cs.heaviest.Height()
+	}
+
+	// Before we look at newTs, let's refresh best tipset at current head's height -- this is done to detect equivocation
+	newHeaviest, newHeaviestWeight, err := cs.FormHeaviestTipSetForHeight(ctx, heaviestHeight)
+	if err != nil {
+		return xerrors.Errorf("failed to reform head at same height: %w", err)
+	}
+
+	// Equivocation has occurred! We need a new head NOW!
+	if newHeaviest == nil || newHeaviestWeight.LessThan(heaviestWeight) {
+		log.Warnf("chainstore heaviest tipset's weight SHRANK from %d (%s) to %d (%s) due to equivocation", heaviestWeight, cs.heaviest, newHeaviestWeight, newHeaviest)
+		// Unfortunately, we don't know what the right height to form a new heaviest tipset is.
+		// It is _probably_, but not _necessarily_, heaviestHeight.
+		// So, we need to explore a range of epochs, finding the heaviest tipset in that range.
+		// We thus try to form the heaviest tipset for 5 epochs above heaviestHeight (most of which will likely not exist),
+		// as well as for 5 below.
+		// This is slow, but we expect to almost-never be here (only if miners are equivocating, which carries a hefty penalty).
+		for i := heaviestHeight + 5; i > heaviestHeight-5; i-- {
+			possibleHeaviestTs, possibleHeaviestWeight, err := cs.FormHeaviestTipSetForHeight(ctx, i)
+			if err != nil {
+				return xerrors.Errorf("failed to produce head at height %d: %w", i, err)
+			}
+
+			if possibleHeaviestWeight.GreaterThan(newHeaviestWeight) {
+				newHeaviestWeight = possibleHeaviestWeight
+				newHeaviest = possibleHeaviestTs
+			}
+		}
+
+		// if we've found something, we know it's the heaviest equivocation-free head, take it IMMEDIATELY
+		if newHeaviest != nil {
+			errTake := cs.takeHeaviestTipSet(ctx, newHeaviest)
+			if errTake != nil {
+				return xerrors.Errorf("failed to take newHeaviest tipset as head: %w", err)
+			}
+		} else {
+			// if we haven't found something, just stay with our equivocation-y head
+			newHeaviest = cs.heaviest
+		}
+	}
+
+	// if the new height we were notified about isn't what we just refreshed at, see if we have a heavier tipset there
+	if newTsHeight != newHeaviest.Height() {
+		bestTs, bestTsWeight, err := cs.FormHeaviestTipSetForHeight(ctx, newTsHeight)
+		if err != nil {
+			return xerrors.Errorf("failed to form new heaviest tipset at height %d: %w", newTsHeight, err)
+		}
+
+		heavier := bestTsWeight.GreaterThan(newHeaviestWeight)
+		if bestTsWeight.Equals(newHeaviestWeight) {
+			heavier = breakWeightTie(bestTs, newHeaviest)
+		}
+
+		if heavier {
+			newHeaviest = bestTs
+		}
+	}
+
+	// Everything's the same as before, exit early
+	if newHeaviest.Equals(cs.heaviest) {
 		return nil
 	}
 
-	w, err := cs.weight(ctx, cs.StateBlockstore(), ts)
+	// At this point, it MUST be true that newHeaviest is heavier than cs.heaviest -- update if fork allows
+	exceeds, err := cs.exceedsForkLength(ctx, cs.heaviest, newHeaviest)
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to check fork length: %w", err)
 	}
-	heaviestW, err := cs.weight(ctx, cs.StateBlockstore(), cs.heaviest)
+
+	if exceeds {
+		return nil
+	}
+
+	err = cs.takeHeaviestTipSet(ctx, newHeaviest)
 	if err != nil {
-		return err
-	}
-
-	heavier := w.GreaterThan(heaviestW)
-	if w.Equals(heaviestW) && !ts.Equals(cs.heaviest) {
-		log.Errorw("weight draw", "currTs", cs.heaviest, "ts", ts)
-		heavier = breakWeightTie(ts, cs.heaviest)
-	}
-
-	if heavier {
-		// TODO: don't do this for initial sync. Now that we don't have a
-		// difference between 'bootstrap sync' and 'caught up' sync, we need
-		// some other heuristic.
-
-		exceeds, err := cs.exceedsForkLength(ctx, cs.heaviest, ts)
-		if err != nil {
-			return err
-		}
-		if exceeds {
-			return nil
-		}
-
-		return cs.takeHeaviestTipSet(ctx, ts)
+		return xerrors.Errorf("failed to take heaviest tipset: %w", err)
 	}
 
 	return nil
@@ -655,6 +689,16 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
 		return err
 	}
 
+	// write the tipsetkey block to the blockstore for EthAPI queries
+	tsBlk, err := ts.Key().ToStorageBlock()
+	if err != nil {
+		return xerrors.Errorf("failed to get tipset key block: %w", err)
+	}
+
+	if err = cs.chainLocalBlockstore.Put(ctx, tsBlk); err != nil {
+		return xerrors.Errorf("failed to put tipset key block: %w", err)
+	}
+
 	if prevHeaviest != nil { // buf
 		if len(cs.reorgCh) > 0 {
 			log.Warnf("Reorg channel running behind, %d reorgs buffered", len(cs.reorgCh))
@@ -904,6 +948,14 @@ func ReorgOps(ctx context.Context, lts func(ctx context.Context, _ types.TipSetK
 
 	var leftChain, rightChain []*types.TipSet
 	for !left.Equals(right) {
+		// this can take a long time and lot of memory if the tipsets are far apart
+		// since it can be reached through remote calls, we need to
+		// cancel early when possible to prevent resource exhaustion.
+		select {
+		case <-ctx.Done():
+			return nil, nil, ctx.Err()
+		default:
+		}
 		if left.Height() > right.Height() {
 			leftChain = append(leftChain, left)
 			par, err := lts(ctx, left.Parents())
@@ -960,7 +1012,7 @@ func (cs *ChainStore) AddToTipSetTracker(ctx context.Context, b *types.BlockHead
 	// This means that we ideally want to keep only most recent 900 epochs in here
 	// Golang's map iteration starts at a random point in a map.
 	// With 5 tries per epoch, and 900 entries to keep, on average we will have
-	// ~136 garbage entires in the `cs.tipsets` map. (solve for 1-(1-x/(900+x))^5 == 0.5)
+	// ~136 garbage entries in the `cs.tipsets` map. (solve for 1-(1-x/(900+x))^5 == 0.5)
 	// Seems good enough to me
 
 	for height := range cs.tipsets {
@@ -975,6 +1027,7 @@ func (cs *ChainStore) AddToTipSetTracker(ctx context.Context, b *types.BlockHead
 	return nil
 }
 
+// PersistTipsets writes the provided blocks and the TipSetKey objects to the blockstore
 func (cs *ChainStore) PersistTipsets(ctx context.Context, tipsets []*types.TipSet) error {
 	toPersist := make([]*types.BlockHeader, 0, len(tipsets)*int(build.BlocksPerEpoch))
 	tsBlks := make([]block.Block, 0, len(tipsets))
@@ -1027,44 +1080,72 @@ func (cs *ChainStore) persistBlockHeaders(ctx context.Context, b ...*types.Block
 	return err
 }
 
-func (cs *ChainStore) expandTipset(ctx context.Context, b *types.BlockHeader) (*types.TipSet, error) {
-	// Hold lock for the whole function for now, if it becomes a problem we can
-	// fix pretty easily
+// FormHeaviestTipSetForHeight looks up all valid blocks at a given height, and returns the heaviest tipset that can be made at that height
+// It does not consider ANY blocks from miners that have "equivocated" (produced 2 blocks at the same height)
+func (cs *ChainStore) FormHeaviestTipSetForHeight(ctx context.Context, height abi.ChainEpoch) (*types.TipSet, types.BigInt, error) {
 	cs.tstLk.Lock()
 	defer cs.tstLk.Unlock()
 
-	all := []*types.BlockHeader{b}
-
-	tsets, ok := cs.tipsets[b.Height]
+	blockCids, ok := cs.tipsets[height]
 	if !ok {
-		return types.NewTipSet(all)
+		return nil, types.NewInt(0), nil
 	}
 
-	inclMiners := map[address.Address]cid.Cid{b.Miner: b.Cid()}
-	for _, bhc := range tsets {
-		if bhc == b.Cid() {
-			continue
-		}
+	// First, identify "bad" miners for the height
 
+	seenMiners := map[address.Address]struct{}{}
+	badMiners := map[address.Address]struct{}{}
+	blocks := make([]*types.BlockHeader, 0, len(blockCids))
+	for _, bhc := range blockCids {
 		h, err := cs.GetBlock(ctx, bhc)
 		if err != nil {
-			return nil, xerrors.Errorf("failed to load block (%s) for tipset expansion: %w", bhc, err)
+			return nil, types.NewInt(0), xerrors.Errorf("failed to load block (%s) for tipset expansion: %w", bhc, err)
 		}
 
-		if cid, found := inclMiners[h.Miner]; found {
-			log.Warnf("Have multiple blocks from miner %s at height %d in our tipset cache %s-%s", h.Miner, h.Height, h.Cid(), cid)
+		if _, seen := seenMiners[h.Miner]; seen {
+			badMiners[h.Miner] = struct{}{}
 			continue
 		}
+		seenMiners[h.Miner] = struct{}{}
+		blocks = append(blocks, h)
+	}
 
-		if types.CidArrsEqual(h.Parents, b.Parents) {
-			all = append(all, h)
-			inclMiners[h.Miner] = bhc
+	// Next, group by parent tipset
+
+	formableTipsets := make(map[types.TipSetKey][]*types.BlockHeader, 0)
+	for _, h := range blocks {
+		if _, bad := badMiners[h.Miner]; bad {
+			continue
+		}
+		ptsk := types.NewTipSetKey(h.Parents...)
+		formableTipsets[ptsk] = append(formableTipsets[ptsk], h)
+	}
+
+	maxWeight := types.NewInt(0)
+	var maxTs *types.TipSet
+	for _, headers := range formableTipsets {
+		ts, err := types.NewTipSet(headers)
+		if err != nil {
+			return nil, types.NewInt(0), xerrors.Errorf("unexpected error forming tipset: %w", err)
+		}
+
+		weight, err := cs.Weight(ctx, ts)
+		if err != nil {
+			return nil, types.NewInt(0), xerrors.Errorf("failed to calculate weight: %w", err)
+		}
+
+		heavier := weight.GreaterThan(maxWeight)
+		if weight.Equals(maxWeight) {
+			heavier = breakWeightTie(ts, maxTs)
+		}
+
+		if heavier {
+			maxWeight = weight
+			maxTs = ts
 		}
 	}
 
-	// TODO: other validation...?
-
-	return types.NewTipSet(all)
+	return maxTs, maxWeight, nil
 }
 
 func (cs *ChainStore) GetGenesis(ctx context.Context) (*types.BlockHeader, error) {

chain/store/store_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/stretchr/testify/require"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 
@@ -238,3 +239,171 @@ func TestChainExportImportFull(t *testing.T) {
 		}
 	}
 }
+
+func TestEquivocations(t *testing.T) {
+	ctx := context.Background()
+	cg, err := gen.NewGenerator()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var last *types.TipSet
+	for i := 0; i < 10; i++ {
+		ts, err := cg.NextTipSet()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		last = ts.TipSet.TipSet()
+	}
+
+	mTs, err := cg.NextTipSetFromMiners(last, []address.Address{last.Blocks()[0].Miner}, 0)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(mTs.TipSet.TipSet().Cids()))
+	last = mTs.TipSet.TipSet()
+
+	require.NotEmpty(t, last.Blocks())
+	blk1 := *last.Blocks()[0]
+
+	// quick check: asking to form tipset at latest height just returns head
+	bestHead, bestHeadWeight, err := cg.ChainStore().FormHeaviestTipSetForHeight(ctx, last.Height())
+	require.NoError(t, err)
+	require.Equal(t, last.Key(), bestHead.Key())
+	require.Contains(t, last.Cids(), blk1.Cid())
+	expectedWeight, err := cg.ChainStore().Weight(ctx, bestHead)
+	require.NoError(t, err)
+	require.Equal(t, expectedWeight, bestHeadWeight)
+
+	// add another block by a different miner -- it should get included in the best tipset
+	blk2 := blk1
+	blk1Miner, err := address.IDFromAddress(blk2.Miner)
+	require.NoError(t, err)
+	blk2.Miner, err = address.NewIDAddress(blk1Miner + 50)
+	require.NoError(t, err)
+	addBlockToTracker(t, cg.ChainStore(), &blk2)
+
+	bestHead, bestHeadWeight, err = cg.ChainStore().FormHeaviestTipSetForHeight(ctx, last.Height())
+	require.NoError(t, err)
+	for _, blkCid := range last.Cids() {
+		require.Contains(t, bestHead.Cids(), blkCid)
+	}
+	require.Contains(t, bestHead.Cids(), blk2.Cid())
+	expectedWeight, err = cg.ChainStore().Weight(ctx, bestHead)
+	require.NoError(t, err)
+	require.Equal(t, expectedWeight, bestHeadWeight)
+
+	// add another block by a different miner, but on a different tipset -- it should NOT get included
+	blk3 := blk1
+	blk3.Miner, err = address.NewIDAddress(blk1Miner + 100)
+	require.NoError(t, err)
+	blk1Parent, err := cg.ChainStore().GetBlock(ctx, blk3.Parents[0])
+	require.NoError(t, err)
+	blk3.Parents = blk1Parent.Parents
+	addBlockToTracker(t, cg.ChainStore(), &blk3)
+
+	bestHead, bestHeadWeight, err = cg.ChainStore().FormHeaviestTipSetForHeight(ctx, last.Height())
+	require.NoError(t, err)
+	for _, blkCid := range last.Cids() {
+		require.Contains(t, bestHead.Cids(), blkCid)
+	}
+	require.Contains(t, bestHead.Cids(), blk2.Cid())
+	require.NotContains(t, bestHead.Cids(), blk3.Cid())
+	expectedWeight, err = cg.ChainStore().Weight(ctx, bestHead)
+	require.NoError(t, err)
+	require.Equal(t, expectedWeight, bestHeadWeight)
+
+	// add another block by the same miner as blk1 -- it should NOT get included, and blk1 should be excluded too
+	blk4 := blk1
+	blk4.Timestamp = blk1.Timestamp + 1
+	addBlockToTracker(t, cg.ChainStore(), &blk4)
+
+	bestHead, bestHeadWeight, err = cg.ChainStore().FormHeaviestTipSetForHeight(ctx, last.Height())
+	require.NoError(t, err)
+	for _, blkCid := range last.Cids() {
+		if blkCid != blk1.Cid() {
+			require.Contains(t, bestHead.Cids(), blkCid)
+		}
+	}
+	require.NotContains(t, bestHead.Cids(), blk4.Cid())
+	require.NotContains(t, bestHead.Cids(), blk1.Cid())
+	expectedWeight, err = cg.ChainStore().Weight(ctx, bestHead)
+	require.NoError(t, err)
+	require.Equal(t, expectedWeight, bestHeadWeight)
+
+	// check that after all of that, the chainstore's head has NOT changed
+	require.Equal(t, last.Key(), cg.ChainStore().GetHeaviestTipSet().Key())
+
+	// NOW, after all that, notify the chainstore to refresh its head
+	require.NoError(t, cg.ChainStore().RefreshHeaviestTipSet(ctx, blk1.Height+1))
+
+	originalHead := *last
+	newHead := cg.ChainStore().GetHeaviestTipSet()
+	// the newHead should be at the same height as the originalHead
+	require.Equal(t, originalHead.Height(), newHead.Height())
+	// the newHead should NOT be the same as the originalHead
+	require.NotEqual(t, originalHead.Key(), newHead.Key())
+	// specifically, it should not contain any blocks by blk1Miner
+	for _, b := range newHead.Blocks() {
+		require.NotEqual(t, blk1.Miner, b.Miner)
+	}
+
+	// now have blk2's Miner equivocate too! this causes us to switch to a tipset with a different parent!
+	blk5 := blk2
+	blk5.Timestamp = blk5.Timestamp + 1
+	addBlockToTracker(t, cg.ChainStore(), &blk5)
+
+	// notify the chainstore to refresh its head
+	require.NoError(t, cg.ChainStore().RefreshHeaviestTipSet(ctx, blk1.Height+1))
+	newHead = cg.ChainStore().GetHeaviestTipSet()
+	// the newHead should still be at the same height as the originalHead
+	require.Equal(t, originalHead.Height(), newHead.Height())
+	// BUT it should no longer have the same parents -- only blk3's miner is good, and they mined on a different tipset
+	require.Equal(t, 1, len(newHead.Blocks()))
+	require.Equal(t, blk3.Cid(), newHead.Cids()[0])
+	require.NotEqual(t, originalHead.Parents(), newHead.Parents())
+
+	// now have blk3's Miner equivocate too! this causes us to switch to a previous epoch entirely :(
+	blk6 := blk3
+	blk6.Timestamp = blk6.Timestamp + 1
+	addBlockToTracker(t, cg.ChainStore(), &blk6)
+
+	// trying to form a tipset at our previous height leads to emptiness
+	tryTs, tryTsWeight, err := cg.ChainStore().FormHeaviestTipSetForHeight(ctx, blk1.Height)
+	require.NoError(t, err)
+	require.Nil(t, tryTs)
+	require.True(t, tryTsWeight.IsZero())
+
+	// notify the chainstore to refresh its head
+	require.NoError(t, cg.ChainStore().RefreshHeaviestTipSet(ctx, blk1.Height+1))
+	newHead = cg.ChainStore().GetHeaviestTipSet()
+	// the newHead should now be one epoch behind originalHead
+	require.Greater(t, originalHead.Height(), newHead.Height())
+
+	// next, we create a new tipset with only one block after many null rounds
+	headAfterNulls, err := cg.NextTipSetFromMiners(newHead, []address.Address{newHead.Blocks()[0].Miner}, 15)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(headAfterNulls.TipSet.Blocks))
+
+	// now, we disqualify the block in this tipset because of equivocation
+	blkAfterNulls := headAfterNulls.TipSet.TipSet().Blocks()[0]
+	equivocatedBlkAfterNulls := *blkAfterNulls
+	equivocatedBlkAfterNulls.Timestamp = blkAfterNulls.Timestamp + 1
+	addBlockToTracker(t, cg.ChainStore(), &equivocatedBlkAfterNulls)
+
+	// try to form a tipset at this height -- it should be empty
+	tryTs2, tryTsWeight2, err := cg.ChainStore().FormHeaviestTipSetForHeight(ctx, blkAfterNulls.Height)
+	require.NoError(t, err)
+	require.Nil(t, tryTs2)
+	require.True(t, tryTsWeight2.IsZero())
+
+	// now we "notify" at this height -- it should lead to no head change because there's no formable head in near epochs
+	require.NoError(t, cg.ChainStore().RefreshHeaviestTipSet(ctx, blkAfterNulls.Height))
+	require.True(t, headAfterNulls.TipSet.TipSet().Equals(cg.ChainStore().GetHeaviestTipSet()))
+}
+
+func addBlockToTracker(t *testing.T, cs *store.ChainStore, blk *types.BlockHeader) {
+	blk2Ts, err := types.NewTipSet([]*types.BlockHeader{blk})
+	require.NoError(t, err)
+	require.NoError(t, cs.PersistTipsets(context.TODO(), []*types.TipSet{blk2Ts}))
+	require.NoError(t, cs.AddToTipSetTracker(context.TODO(), blk))
+}

chain/sub/incoming.go

@@ -350,6 +350,7 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
 		)
 		recordFailure(ctx, metrics.MessageValidationFailure, "add")
 		switch {
+
 		case xerrors.Is(err, messagepool.ErrSoftValidationFailure):
 			fallthrough
 		case xerrors.Is(err, messagepool.ErrRBFTooLowPremium):
@@ -362,8 +363,17 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
 			fallthrough
 		case xerrors.Is(err, messagepool.ErrNonceTooLow):
 			fallthrough
+		case xerrors.Is(err, messagepool.ErrNotEnoughFunds):
+			fallthrough
 		case xerrors.Is(err, messagepool.ErrExistingNonce):
 			return pubsub.ValidationIgnore
+
+		case xerrors.Is(err, messagepool.ErrMessageTooBig):
+			fallthrough
+		case xerrors.Is(err, messagepool.ErrMessageValueTooHigh):
+			fallthrough
+		case xerrors.Is(err, messagepool.ErrInvalidToAddr):
+			fallthrough
 		default:
 			return pubsub.ValidationReject
 		}
@@ -519,9 +529,8 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 
 	msgCid := idxrMsg.Cid
 
-	var msgInfo *peerMsgInfo
-	msgInfo, ok := v.peerCache.Get(minerAddr)
-	if !ok {
+	msgInfo, cached := v.peerCache.Get(minerAddr)
+	if !cached {
 		msgInfo = &peerMsgInfo{}
 	}
 
@@ -529,17 +538,17 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 	msgInfo.mutex.Lock()
 	defer msgInfo.mutex.Unlock()
 
-	if ok {
+	var seqno uint64
+	if cached {
 		// Reject replayed messages.
-		seqno := binary.BigEndian.Uint64(msg.Message.GetSeqno())
+		seqno = binary.BigEndian.Uint64(msg.Message.GetSeqno())
 		if seqno <= msgInfo.lastSeqno {
 			log.Debugf("ignoring replayed indexer message")
 			return pubsub.ValidationIgnore
 		}
-		msgInfo.lastSeqno = seqno
 	}
 
-	if !ok || originPeer != msgInfo.peerID {
+	if !cached || originPeer != msgInfo.peerID {
 		// Check that the miner ID maps to the peer that sent the message.
 		err = v.authenticateMessage(ctx, minerAddr, originPeer)
 		if err != nil {
@@ -548,7 +557,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 			return pubsub.ValidationReject
 		}
 		msgInfo.peerID = originPeer
-		if !ok {
+		if !cached {
 			// Add msgInfo to cache only after being authenticated.  If two
 			// messages from the same peer are handled concurrently, there is a
 			// small chance that one msgInfo could replace the other here when
@@ -557,6 +566,9 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 		}
 	}
 
+	// Update message info cache with the latest message's sequence number.
+	msgInfo.lastSeqno = seqno
+
 	// See if message needs to be ignored due to rate limiting.
 	if v.rateLimitPeer(msgInfo, msgCid) {
 		return pubsub.ValidationIgnore

chain/sub/incoming_test.go

@@ -12,10 +12,12 @@ import (
 	"github.com/ipni/go-libipni/announce/message"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pb "github.com/libp2p/go-libp2p-pubsub/pb"
+	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 
 	"github.com/filecoin-project/go-address"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/mocks"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -134,3 +136,123 @@ func TestIndexerMessageValidator_Validate(t *testing.T) {
 		})
 	}
 }
+
+func TestIdxValidator(t *testing.T) {
+	validCid, err := cid.Decode("QmbpDgg5kRLDgMxS8vPKNFXEcA6D5MC4CkuUdSWDVtHPGK")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	addr, err := address.NewFromString("f01024")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf1, err := addr.MarshalBinary()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	selfPID := "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW"
+	senderPID := "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ"
+	extraData := buf1
+
+	mc := gomock.NewController(t)
+	node := mocks.NewMockFullNode(mc)
+	node.EXPECT().ChainHead(gomock.Any()).Return(nil, nil).AnyTimes()
+
+	subject := NewIndexerMessageValidator(peer.ID(selfPID), node, node)
+	message := message.Message{
+		Cid:       validCid,
+		Addrs:     nil,
+		ExtraData: extraData,
+	}
+	buf := bytes.NewBuffer(nil)
+	if err := message.MarshalCBOR(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	topic := "topic"
+
+	privk, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id, err := peer.IDFromPublicKey(privk.GetPublic())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	node.EXPECT().StateMinerInfo(gomock.Any(), gomock.Any(), gomock.Any()).Return(api.MinerInfo{PeerId: &id}, nil).AnyTimes()
+
+	pbm := &pb.Message{
+		Data:  buf.Bytes(),
+		Topic: &topic,
+		From:  []byte(id),
+		Seqno: []byte{1, 1, 1, 1, 2, 2, 2, 2},
+	}
+	validate := subject.Validate(context.Background(), peer.ID(senderPID), &pubsub.Message{
+		Message:       pbm,
+		ReceivedFrom:  peer.ID("f01024"), // peer.ID(senderPID),
+		ValidatorData: nil,
+	})
+	if validate != pubsub.ValidationAccept {
+		t.Error("Expected to receive ValidationAccept")
+	}
+	msgInfo, cached := subject.peerCache.Get(addr)
+	if !cached {
+		t.Fatal("Message info should be in cache")
+	}
+	seqno := msgInfo.lastSeqno
+	msgInfo.rateLimit = nil // prevent interference from rate limiting
+
+	t.Log("Sending DoS msg")
+	privk, _, err = crypto.GenerateKeyPair(crypto.RSA, 2048)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := peer.IDFromPublicKey(privk.GetPublic())
+	if err != nil {
+		t.Fatal(err)
+	}
+	pbm = &pb.Message{
+		Data:  buf.Bytes(),
+		Topic: &topic,
+		From:  []byte(id2),
+		Seqno: []byte{255, 255, 255, 255, 255, 255, 255, 255},
+	}
+	validate = subject.Validate(context.Background(), peer.ID(senderPID), &pubsub.Message{
+		Message:       pbm,
+		ReceivedFrom:  peer.ID(senderPID),
+		ValidatorData: nil,
+	})
+	if validate != pubsub.ValidationReject {
+		t.Error("Expected to get ValidationReject")
+	}
+	msgInfo, cached = subject.peerCache.Get(addr)
+	if !cached {
+		t.Fatal("Message info should be in cache")
+	}
+	msgInfo.rateLimit = nil // prevent interference from rate limiting
+
+	// Check if DoS is possible.
+	if msgInfo.lastSeqno != seqno {
+		t.Fatal("Sequence number should not have been updated")
+	}
+
+	t.Log("Sending another valid message from miner...")
+	pbm = &pb.Message{
+		Data:  buf.Bytes(),
+		Topic: &topic,
+		From:  []byte(id),
+		Seqno: []byte{1, 1, 1, 1, 2, 2, 2, 3},
+	}
+	validate = subject.Validate(context.Background(), peer.ID(senderPID), &pubsub.Message{
+		Message:       pbm,
+		ReceivedFrom:  peer.ID("f01024"), // peer.ID(senderPID),
+		ValidatorData: nil,
+	})
+	if validate != pubsub.ValidationAccept {
+		t.Fatal("Did not receive ValidationAccept")
+	}
+}

chain/sync.go

@@ -536,7 +536,7 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error {
 
 	// At this point we have accepted and synced to the new `maybeHead`
 	// (`StageSyncComplete`).
-	if err := syncer.store.PutTipSet(ctx, maybeHead); err != nil {
+	if err := syncer.store.RefreshHeaviestTipSet(ctx, maybeHead.Height()); err != nil {
 		span.AddAttributes(trace.StringAttribute("put_error", err.Error()))
 		span.SetStatus(trace.Status{
 			Code:    13,
@@ -1094,8 +1094,8 @@ func (syncer *Syncer) fetchMessages(ctx context.Context, headers []*types.TipSet
 						requestErr = multierror.Append(requestErr, err)
 					} else {
 						isGood := true
-						for index, ts := range headers[nextI:lastI] {
-							cm := result[index]
+						for index, cm := range result {
+							ts := headers[nextI+index]
 							if err := checkMsgMeta(ts, cm.Bls, cm.Secpk, cm.BlsIncludes, cm.SecpkIncludes); err != nil {
 								log.Errorf("fetched messages not as expected: %s", err)
 								isGood = false

chain/sync_manager.go

@@ -92,6 +92,7 @@ type syncManager struct {
 var _ SyncManager = (*syncManager)(nil)
 
 type peerHead struct {
+	// Note: this doesn't _necessarily_ mean that p's head is ts, just that ts is a tipset that p sent to us
 	p  peer.ID
 	ts *types.TipSet
 }

chain/sync_test.go

@@ -311,7 +311,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) {
 	for _, lastB := range lastTs.Blocks {
 		require.NoError(tu.t, cs.AddToTipSetTracker(context.Background(), lastB.Header))
 	}
-	err = cs.PutTipSet(tu.ctx, lastTs.TipSet())
+	err = cs.RefreshHeaviestTipSet(tu.ctx, lastTs.TipSet().Height())
 	require.NoError(tu.t, err)
 
 	tu.genesis = genesis

chain/types/ethtypes/eth_types.go

@@ -18,6 +18,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	builtintypes "github.com/filecoin-project/go-state-types/builtin"
 
@@ -798,6 +799,45 @@ func GetContractEthAddressFromCode(sender EthAddress, salt [32]byte, initcode []
 	return ethAddr, nil
 }
 
+// EthEstimateGasParams handles raw jsonrpc params for eth_estimateGas
+type EthEstimateGasParams struct {
+	Tx       EthCall
+	BlkParam *EthBlockNumberOrHash
+}
+
+func (e *EthEstimateGasParams) UnmarshalJSON(b []byte) error {
+	var params []json.RawMessage
+	err := json.Unmarshal(b, &params)
+	if err != nil {
+		return err
+	}
+
+	switch len(params) {
+	case 2:
+		err = json.Unmarshal(params[1], &e.BlkParam)
+		if err != nil {
+			return err
+		}
+		fallthrough
+	case 1:
+		err = json.Unmarshal(params[0], &e.Tx)
+		if err != nil {
+			return err
+		}
+	default:
+		return xerrors.Errorf("expected 1 or 2 params, got %d", len(params))
+	}
+
+	return nil
+}
+
+func (e EthEstimateGasParams) MarshalJSON() ([]byte, error) {
+	if e.BlkParam != nil {
+		return json.Marshal([]interface{}{e.Tx, e.BlkParam})
+	}
+	return json.Marshal([]interface{}{e.Tx})
+}
+
 // EthFeeHistoryParams handles raw jsonrpc params for eth_feeHistory
 type EthFeeHistoryParams struct {
 	BlkCount          EthUint64
@@ -929,3 +969,57 @@ func (e *EthBlockNumberOrHash) UnmarshalJSON(b []byte) error {
 
 	return errors.New("invalid block param")
 }
+
+type EthTrace struct {
+	Action       EthTraceAction `json:"action"`
+	Result       EthTraceResult `json:"result"`
+	Subtraces    int            `json:"subtraces"`
+	TraceAddress []int          `json:"traceAddress"`
+	Type         string         `json:"Type"`
+
+	Parent *EthTrace `json:"-"`
+
+	// if a subtrace makes a call to GetBytecode, we store a pointer to that subtrace here
+	// which we then lookup when checking for delegatecall (InvokeContractDelegate)
+	LastByteCode *EthTrace `json:"-"`
+}
+
+func (t *EthTrace) SetCallType(callType string) {
+	t.Action.CallType = callType
+	t.Type = callType
+}
+
+type EthTraceBlock struct {
+	*EthTrace
+	BlockHash           EthHash `json:"blockHash"`
+	BlockNumber         int64   `json:"blockNumber"`
+	TransactionHash     EthHash `json:"transactionHash"`
+	TransactionPosition int     `json:"transactionPosition"`
+}
+
+type EthTraceReplayBlockTransaction struct {
+	Output          EthBytes    `json:"output"`
+	StateDiff       *string     `json:"stateDiff"`
+	Trace           []*EthTrace `json:"trace"`
+	TransactionHash EthHash     `json:"transactionHash"`
+	VmTrace         *string     `json:"vmTrace"`
+}
+
+type EthTraceAction struct {
+	CallType string     `json:"callType"`
+	From     EthAddress `json:"from"`
+	To       EthAddress `json:"to"`
+	Gas      EthUint64  `json:"gas"`
+	Input    EthBytes   `json:"input"`
+	Value    EthBigInt  `json:"value"`
+
+	FilecoinMethod  abi.MethodNum   `json:"-"`
+	FilecoinCodeCid cid.Cid         `json:"-"`
+	FilecoinFrom    address.Address `json:"-"`
+	FilecoinTo      address.Address `json:"-"`
+}
+
+type EthTraceResult struct {
+	GasUsed EthUint64 `json:"gasUsed"`
+	Output  EthBytes  `json:"output"`
+}

chain/types/ethtypes/rlp.go

@@ -134,7 +134,7 @@ func decodeRLP(data []byte) (res interface{}, consumed int, err error) {
 			return nil, 0, err
 		}
 		totalLen := 1 + strLenInBytes + strLen
-		if totalLen > len(data) {
+		if totalLen > len(data) || totalLen < 0 {
 			return nil, 0, xerrors.Errorf("invalid rlp data: out of bound while parsing string")
 		}
 		return data[1+strLenInBytes : totalLen], totalLen, nil
@@ -157,7 +157,12 @@ func decodeLength(data []byte, lenInBytes int) (length int, err error) {
 	if err := binary.Read(r, binary.BigEndian, &decodedLength); err != nil {
 		return 0, xerrors.Errorf("invalid rlp data: cannot parse string length: %w", err)
 	}
-	if lenInBytes+int(decodedLength) > len(data) {
+	if decodedLength < 0 {
+		return 0, xerrors.Errorf("invalid rlp data: negative string length")
+	}
+
+	totalLength := lenInBytes + int(decodedLength)
+	if totalLength < 0 || totalLength > len(data) {
 		return 0, xerrors.Errorf("invalid rlp data: out of bound while parsing list")
 	}
 	return int(decodedLength), nil

chain/types/ethtypes/rlp_test.go

@@ -143,6 +143,20 @@ func TestDecodeList(t *testing.T) {
 	}
 }
 
+func TestDecodeNegativeLength(t *testing.T) {
+	testcases := [][]byte{
+		mustDecodeHex("0xbfffffffffffffff0041424344"),
+		mustDecodeHex("0xc1bFFF1111111111111111"),
+		mustDecodeHex("0xbFFF11111111111111"),
+		mustDecodeHex("0xbf7fffffffffffffff41424344"),
+	}
+
+	for _, tc := range testcases {
+		_, err := DecodeRLP(tc)
+		require.ErrorContains(t, err, "invalid rlp data")
+	}
+}
+
 func TestDecodeEncodeTx(t *testing.T) {
 	testcases := [][]byte{
 		mustDecodeHex("0xdc82013a0185012a05f2008504a817c8008080872386f26fc1000000c0"),

chain/types/event.go

@@ -1,11 +1,6 @@
 package types
 
 import (
-	"bytes"
-	"fmt"
-
-	cbg "github.com/whyrusleeping/cbor-gen"
-
 	"github.com/filecoin-project/go-state-types/abi"
 )
 
@@ -38,24 +33,3 @@ type EventEntry struct {
 }
 
 type FilterID [32]byte // compatible with EthHash
-
-// DecodeEvents decodes a CBOR list of CBOR-encoded events.
-func DecodeEvents(input []byte) ([]Event, error) {
-	r := bytes.NewReader(input)
-	typ, len, err := cbg.NewCborReader(r).ReadHeader()
-	if err != nil {
-		return nil, fmt.Errorf("failed to read events: %w", err)
-	}
-	if typ != cbg.MajArray {
-		return nil, fmt.Errorf("expected a CBOR list, was major type %d", typ)
-	}
-	events := make([]Event, 0, len)
-	for i := 0; i < int(len); i++ {
-		var evt Event
-		if err := evt.UnmarshalCBOR(r); err != nil {
-			return nil, fmt.Errorf("failed to parse event: %w", err)
-		}
-		events = append(events, evt)
-	}
-	return events, nil
-}

chain/vm/fvm.go

@@ -92,6 +92,15 @@ func (x *FvmExtern) VerifyConsensusFault(ctx context.Context, a, b, extra []byte
 		log.Info("invalid consensus fault: submitted blocks are the same")
 		return ret, totalGas
 	}
+
+	// workaround chain halt
+	if build.IsNearUpgrade(blockA.Height, build.UpgradeWatermelonFixHeight) {
+		return ret, totalGas
+	}
+	if build.IsNearUpgrade(blockB.Height, build.UpgradeWatermelonFixHeight) {
+		return ret, totalGas
+	}
+
 	// (1) check conditions necessary to any consensus fault
 
 	// were blocks mined by same miner?
@@ -459,7 +468,7 @@ func (vm *FVM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet
 	}
 
 	if vm.returnEvents && len(ret.EventsBytes) > 0 {
-		applyRet.Events, err = types.DecodeEvents(ret.EventsBytes)
+		applyRet.Events, err = decodeEvents(ret.EventsBytes)
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode events returned by the FVM: %w", err)
 		}
@@ -515,7 +524,7 @@ func (vm *FVM) ApplyImplicitMessage(ctx context.Context, cmsg *types.Message) (*
 	}
 
 	if vm.returnEvents && len(ret.EventsBytes) > 0 {
-		applyRet.Events, err = types.DecodeEvents(ret.EventsBytes)
+		applyRet.Events, err = decodeEvents(ret.EventsBytes)
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode events returned by the FVM: %w", err)
 		}

chain/vm/fvm_util.go

@@ -0,0 +1,39 @@
+package vm
+
+import (
+	"bytes"
+	"fmt"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+const MaxEventSliceLength = 6_000_000
+
+// DecodeEvents decodes a CBOR list of CBOR-encoded events.
+func decodeEvents(input []byte) ([]types.Event, error) {
+	r := bytes.NewReader(input)
+	typ, length, err := cbg.NewCborReader(r).ReadHeader()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read events: %w", err)
+	}
+
+	if length > MaxEventSliceLength {
+		log.Errorf("extremely long event slice (len %d) returned, not decoding", length)
+		return nil, nil
+	}
+
+	if typ != cbg.MajArray {
+		return nil, fmt.Errorf("expected a CBOR list, was major type %d", typ)
+	}
+	events := make([]types.Event, 0, length)
+	for i := 0; i < int(length); i++ {
+		var evt types.Event
+		if err := evt.UnmarshalCBOR(r); err != nil {
+			return nil, fmt.Errorf("failed to parse event: %w", err)
+		}
+		events = append(events, evt)
+	}
+	return events, nil
+}

chain/vm/syscalls.go

@@ -70,11 +70,6 @@ type syscallShim struct {
 }
 
 func (ss *syscallShim) ComputeUnsealedSectorCID(st abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) {
-	var sum abi.PaddedPieceSize
-	for _, p := range pieces {
-		sum += p.Size
-	}
-
 	commd, err := ffiwrapper.GenerateUnsealedCID(st, pieces)
 	if err != nil {
 		log.Errorf("generate data commitment failed: %s", err)

chain/vm/vm.go

@@ -11,7 +11,7 @@ import (
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
-	mh "github.com/multiformats/go-multihash"
+	"github.com/multiformats/go-multicodec"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
@@ -38,7 +38,6 @@ import (
 )
 
 const MaxCallDepth = 4096
-const CborCodec = 0x51
 
 var (
 	log            = logging.Logger("vm")
@@ -128,7 +127,7 @@ func (bs *gasChargingBlocks) Put(ctx context.Context, blk block.Block) error {
 func (vm *LegacyVM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runtime) *Runtime {
 	paramsCodec := uint64(0)
 	if len(msg.Params) > 0 {
-		paramsCodec = CborCodec
+		paramsCodec = uint64(multicodec.Cbor)
 	}
 	rt := &Runtime{
 		ctx:         ctx,
@@ -380,7 +379,7 @@ func (vm *LegacyVM) send(ctx context.Context, msg *types.Message, parent *Runtim
 
 	retCodec := uint64(0)
 	if len(ret) > 0 {
-		retCodec = CborCodec
+		retCodec = uint64(multicodec.Cbor)
 	}
 	rt.executionTrace.MsgRct = types.ReturnTrace{
 		ExitCode:    aerrors.RetCode(err),
@@ -695,15 +694,15 @@ func (vm *LegacyVM) ActorStore(ctx context.Context) adt.Store {
 }
 
 func linksForObj(blk block.Block, cb func(cid.Cid)) error {
-	switch blk.Cid().Prefix().Codec {
-	case cid.DagCBOR:
+	switch multicodec.Code(blk.Cid().Prefix().Codec) {
+	case multicodec.DagCbor:
 		err := cbg.ScanForLinks(bytes.NewReader(blk.RawData()), cb)
 		if err != nil {
 			return xerrors.Errorf("cbg.ScanForLinks: %w", err)
 		}
 		return nil
-	case cid.Raw:
-		// We implicitly have all children of raw blocks.
+	case multicodec.Raw, multicodec.Cbor:
+		// We implicitly have all children of raw/cbor blocks.
 		return nil
 	default:
 		return xerrors.Errorf("vm flush copy method only supports dag cbor")
@@ -803,14 +802,17 @@ func copyRec(ctx context.Context, from, to blockstore.Blockstore, root cid.Cid,
 		}
 
 		prefix := link.Prefix()
-		if prefix.Codec == cid.FilCommitmentSealed || prefix.Codec == cid.FilCommitmentUnsealed {
+		codec := multicodec.Code(prefix.Codec)
+		switch codec {
+		case multicodec.FilCommitmentSealed, cid.FilCommitmentUnsealed:
 			return
 		}
 
 		// We always have blocks inlined into CIDs, but we may not have their children.
-		if prefix.MhType == mh.IDENTITY {
+		if multicodec.Code(prefix.MhType) == multicodec.Identity {
 			// Unless the inlined block has no children.
-			if prefix.Codec == cid.Raw {
+			switch codec {
+			case multicodec.Raw, multicodec.Cbor:
 				return
 			}
 		} else {

cli/sync.go

@@ -273,11 +273,6 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 			continue
 		}
 
-		head, err := napi.ChainHead(ctx)
-		if err != nil {
-			return err
-		}
-
 		working := -1
 		for i, ss := range state.ActiveSyncs {
 			switch ss.Stage {
@@ -332,7 +327,11 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 
 		_ = target // todo: maybe print? (creates a bunch of line wrapping issues with most tipsets)
 
-		if !watch && time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) {
+		isDone, err := IsSyncDone(ctx, napi)
+		if err != nil {
+			return err
+		}
+		if !watch && isDone {
 			fmt.Println("\nDone!")
 			return nil
 		}
@@ -347,3 +346,11 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 		i++
 	}
 }
+
+func IsSyncDone(ctx context.Context, napi v0api.FullNode) (bool, error) {
+	head, err := napi.ChainHead(ctx)
+	if err != nil {
+		return false, err
+	}
+	return time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs), nil
+}

cli/wallet.go

@@ -7,7 +7,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"
 
 	"github.com/urfave/cli/v2"
 	"golang.org/x/term"
@@ -206,7 +208,12 @@ var walletBalance = &cli.Command{
 			return err
 		}
 
-		if balance.Equals(types.NewInt(0)) {
+		inSync, err := IsSyncDone(ctx, api)
+		if err != nil {
+			return err
+		}
+
+		if balance.Equals(types.NewInt(0)) && !inSync {
 			afmt.Printf("%s (warning: may display 0 if chain sync in progress)\n", types.FIL(balance))
 		} else {
 			afmt.Printf("%s\n", types.FIL(balance))
@@ -330,6 +337,17 @@ var walletImport = &cli.Command{
 		if !cctx.Args().Present() || cctx.Args().First() == "-" {
 			if term.IsTerminal(int(os.Stdin.Fd())) {
 				fmt.Print("Enter private key(not display in the terminal): ")
+
+				sigCh := make(chan os.Signal, 1)
+				// Notify the channel when SIGINT is received
+				signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+				go func() {
+					<-sigCh
+					fmt.Println("\nInterrupt signal received. Exiting...")
+					os.Exit(1)
+				}()
+
 				inpdata, err = term.ReadPassword(int(os.Stdin.Fd()))
 				if err != nil {
 					return err

cli/wallet_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
 )
 
 func TestWalletNew(t *testing.T) {
@@ -133,6 +134,11 @@ func TestWalletBalance(t *testing.T) {
 
 	balance := big.NewInt(1234)
 
+	// add blocks to the chain
+	first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+	head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+	mockApi.EXPECT().ChainHead(ctx).Return(head, nil)
 	mockApi.EXPECT().WalletBalance(ctx, addr).Return(balance, nil)
 
 	//stm: @CLI_WALLET_BALANCE_001

cmd/lotus-bench/amt_internal.go

@@ -0,0 +1,312 @@
+// Copied from go-amt-ipld https://github.com/filecoin-project/go-amt-ipld/tree/master/internal
+// which for some reason is a go internal package and therefore cannot be imported
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"sort"
+
+	cid "github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+)
+
+type AMTRoot struct {
+	BitWidth uint64
+	Height   uint64
+	Count    uint64
+	AMTNode  AMTNode
+}
+
+type AMTNode struct {
+	Bmap   []byte
+	Links  []cid.Cid
+	Values []*cbg.Deferred
+}
+
+// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
+
+var _ = xerrors.Errorf
+var _ = cid.Undef
+var _ = math.E
+var _ = sort.Sort
+
+var lengthBufAMTRoot = []byte{132}
+
+func (t *AMTRoot) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufAMTRoot); err != nil {
+		return err
+	}
+
+	// t.BitWidth (uint64) (uint64)
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, t.BitWidth); err != nil {
+		return err
+	}
+
+	// t.Height (uint64) (uint64)
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, t.Height); err != nil {
+		return err
+	}
+
+	// t.Count (uint64) (uint64)
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, t.Count); err != nil {
+		return err
+	}
+
+	// t.AMTNode (internal.AMTNode) (struct)
+	if err := t.AMTNode.MarshalCBOR(cw); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *AMTRoot) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = AMTRoot{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra != 4 {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	// t.BitWidth (uint64) (uint64)
+
+	{
+
+		maj, extra, err = cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.BitWidth = extra
+
+	}
+	// t.Height (uint64) (uint64)
+
+	{
+
+		maj, extra, err = cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Height = extra
+
+	}
+	// t.Count (uint64) (uint64)
+
+	{
+
+		maj, extra, err = cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Count = extra
+
+	}
+	// t.AMTNode (internal.AMTNode) (struct)
+
+	{
+
+		if err := t.AMTNode.UnmarshalCBOR(cr); err != nil {
+			return xerrors.Errorf("unmarshaling t.AMTNode: %w", err)
+		}
+
+	}
+	return nil
+}
+
+var lengthBufAMTNode = []byte{131}
+
+func (t *AMTNode) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufAMTNode); err != nil {
+		return err
+	}
+
+	// t.Bmap ([]uint8) (slice)
+	if len(t.Bmap) > cbg.ByteArrayMaxLen {
+		return xerrors.Errorf("Byte array in field t.Bmap was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Bmap))); err != nil {
+		return err
+	}
+
+	if _, err := cw.Write(t.Bmap[:]); err != nil {
+		return err
+	}
+
+	// t.Links ([]cid.Cid) (slice)
+	if len(t.Links) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Links was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Links))); err != nil {
+		return err
+	}
+	for _, v := range t.Links {
+		if err := cbg.WriteCid(w, v); err != nil {
+			return xerrors.Errorf("failed writing cid field t.Links: %w", err)
+		}
+	}
+
+	// t.Values ([]*typegen.Deferred) (slice)
+	if len(t.Values) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Values was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Values))); err != nil {
+		return err
+	}
+	for _, v := range t.Values {
+		if err := v.MarshalCBOR(cw); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *AMTNode) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = AMTNode{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra != 3 {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	// t.Bmap ([]uint8) (slice)
+
+	maj, extra, err = cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.ByteArrayMaxLen {
+		return fmt.Errorf("t.Bmap: byte array too large (%d)", extra)
+	}
+	if maj != cbg.MajByteString {
+		return fmt.Errorf("expected byte array")
+	}
+
+	if extra > 0 {
+		t.Bmap = make([]uint8, extra)
+	}
+
+	if _, err := io.ReadFull(cr, t.Bmap[:]); err != nil {
+		return err
+	}
+	// t.Links ([]cid.Cid) (slice)
+
+	maj, extra, err = cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.Links: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.Links = make([]cid.Cid, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+
+		c, err := cbg.ReadCid(cr)
+		if err != nil {
+			return xerrors.Errorf("reading cid field t.Links failed: %w", err)
+		}
+		t.Links[i] = c
+	}
+
+	// t.Values ([]*typegen.Deferred) (slice)
+
+	maj, extra, err = cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("t.Values: array too large (%d)", extra)
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("expected cbor array")
+	}
+
+	if extra > 0 {
+		t.Values = make([]*cbg.Deferred, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+
+		var v cbg.Deferred
+		if err := v.UnmarshalCBOR(cr); err != nil {
+			return err
+		}
+
+		t.Values[i] = &v
+	}
+
+	return nil
+}

cmd/lotus-bench/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -8,9 +9,16 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/docker/go-units"
+	"github.com/ipfs/boxo/blockservice"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	format "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/minio/blake2b-simd"
 	"github.com/mitchellh/go-homedir"
@@ -20,10 +28,14 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-paramfetch"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	prooftypes "github.com/filecoin-project/go-state-types/proof"
+	adt "github.com/filecoin-project/specs-actors/v6/actors/util/adt"
 
 	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -104,6 +116,7 @@ func main() {
 		DisableSliceFlagSeparator: true,
 		Commands: []*cli.Command{
 			proveCmd,
+			amtBenchCmd,
 			sealBenchCmd,
 			simpleCmd,
 			importBenchCmd,
@@ -117,6 +130,211 @@ func main() {
 	}
 }
 
+type amtStatCollector struct {
+	ds   format.NodeGetter
+	walk func(format.Node) ([]*format.Link, error)
+
+	statsLk               sync.Mutex
+	totalAMTLinks         int
+	totalAMTValues        int
+	totalAMTLinkNodes     int
+	totalAMTValueNodes    int
+	totalAMTLinkNodeSize  int
+	totalAMTValueNodeSize int
+}
+
+func (asc *amtStatCollector) String() string {
+	asc.statsLk.Lock()
+	defer asc.statsLk.Unlock()
+
+	str := "\n------------\n"
+	str += fmt.Sprintf("Link Count: %d\n", asc.totalAMTLinks)
+	str += fmt.Sprintf("Value Count: %d\n", asc.totalAMTValues)
+	str += fmt.Sprintf("%d link nodes %d bytes\n", asc.totalAMTLinkNodes, asc.totalAMTLinkNodeSize)
+	str += fmt.Sprintf("%d value nodes %d bytes\n", asc.totalAMTValueNodes, asc.totalAMTValueNodeSize)
+	str += fmt.Sprintf("Total bytes: %d\n------------\n", asc.totalAMTLinkNodeSize+asc.totalAMTValueNodeSize)
+	return str
+}
+
+func (asc *amtStatCollector) record(ctx context.Context, nd format.Node) error {
+	size, err := nd.Size()
+	if err != nil {
+		return err
+	}
+
+	var node AMTNode
+	if err := node.UnmarshalCBOR(bytes.NewReader(nd.RawData())); err != nil {
+		// try to deserialize root
+		var root AMTRoot
+		if err := root.UnmarshalCBOR(bytes.NewReader(nd.RawData())); err != nil {
+			return err
+		}
+		node = root.AMTNode
+	}
+
+	asc.statsLk.Lock()
+	defer asc.statsLk.Unlock()
+
+	link := len(node.Links) > 0
+	value := len(node.Values) > 0
+
+	if link {
+		asc.totalAMTLinks += len(node.Links)
+		asc.totalAMTLinkNodes++
+		asc.totalAMTLinkNodeSize += int(size)
+	} else if value {
+		asc.totalAMTValues += len(node.Values)
+		asc.totalAMTValueNodes++
+		asc.totalAMTValueNodeSize += int(size)
+	} else {
+		return xerrors.Errorf("unexpected AMT node %x: neither link nor value", nd.RawData())
+	}
+
+	return nil
+}
+
+func (asc *amtStatCollector) walkLinks(ctx context.Context, c cid.Cid) ([]*format.Link, error) {
+	nd, err := asc.ds.Get(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := asc.record(ctx, nd); err != nil {
+		return nil, err
+	}
+
+	return asc.walk(nd)
+}
+
+func carWalkFunc(nd format.Node) (out []*format.Link, err error) {
+	for _, link := range nd.Links() {
+		if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+			continue
+		}
+		out = append(out, link)
+	}
+	return out, nil
+}
+
+var amtBenchCmd = &cli.Command{
+	Name:  "amt",
+	Usage: "Benchmark AMT churn",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "rounds",
+			Usage: "rounds of churn to measure",
+			Value: 1,
+		},
+		&cli.IntFlag{
+			Name:  "interval",
+			Usage: "AMT idx interval for churning values",
+			Value: 2880,
+		},
+		&cli.IntFlag{
+			Name:  "bitwidth",
+			Usage: "AMT bitwidth",
+			Value: 6,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		bs := blockstore.NewMemory()
+		ctx := c.Context
+		store := adt.WrapStore(ctx, cbor.NewCborStore(bs))
+
+		// Setup in memory blockstore
+		bitwidth := c.Int("bitwidth")
+		array, err := adt.MakeEmptyArray(store, bitwidth)
+		if err != nil {
+			return err
+		}
+
+		// Using motivating empirical example: market actor states AMT
+		// Create 40,000,000 states for realistic workload
+		fmt.Printf("Populating AMT\n")
+		for i := 0; i < 40000000; i++ {
+			if err := array.Set(uint64(i), &market.DealState{
+				SectorStartEpoch: abi.ChainEpoch(2000000 + i),
+				LastUpdatedEpoch: abi.ChainEpoch(-1),
+				SlashEpoch:       -1,
+				VerifiedClaim:    verifreg.AllocationId(i),
+			}); err != nil {
+				return err
+			}
+		}
+
+		r, err := array.Root()
+		if err != nil {
+			return err
+		}
+
+		// Measure ratio of internal / leaf nodes / sizes
+		dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+		asc := &amtStatCollector{
+			ds:   dag,
+			walk: carWalkFunc,
+		}
+
+		fmt.Printf("Measuring AMT\n")
+		seen := cid.NewSet()
+		if err := merkledag.Walk(ctx, asc.walkLinks, r, seen.Visit, merkledag.Concurrent()); err != nil {
+			return err
+		}
+
+		fmt.Printf("%s\n", asc)
+
+		// Overwrite ids with idx % interval: one epoch of market cron
+		rounds := c.Int("rounds")
+		interval := c.Int("interval")
+
+		fmt.Printf("Overwrite 1 out of %d values for %d rounds\n", interval, rounds)
+		array, err = adt.AsArray(store, r, bitwidth)
+		if err != nil {
+			return err
+		}
+		roots := make([]cid.Cid, rounds)
+		for j := 0; j < rounds; j++ {
+			if j%10 == 0 {
+				fmt.Printf("round: %d\n", j)
+			}
+			for i := j; i < 40000000; i += interval {
+				if i%interval == j {
+					if err := array.Set(uint64(i), &market.DealState{
+						SectorStartEpoch: abi.ChainEpoch(2000000 + i),
+						LastUpdatedEpoch: abi.ChainEpoch(1),
+						SlashEpoch:       -1,
+						VerifiedClaim:    verifreg.AllocationId(i),
+					}); err != nil {
+						return err
+					}
+				}
+			}
+			roots[j], err = array.Root()
+			if err != nil {
+				return err
+			}
+
+		}
+
+		// Measure churn
+		dag = merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+		asc = &amtStatCollector{
+			ds:   dag,
+			walk: carWalkFunc,
+		}
+
+		fmt.Printf("Measuring %d rounds of churn\n", rounds)
+
+		for _, r := range roots {
+			if err := merkledag.Walk(ctx, asc.walkLinks, r, seen.Visit, merkledag.Concurrent()); err != nil {
+				return err
+			}
+		}
+
+		fmt.Printf("%s\n", asc)
+		return nil
+	},
+}
+
 var sealBenchCmd = &cli.Command{
 	Name:  "sealing",
 	Usage: "Benchmark seal and winning post and window post",

cmd/lotus-bench/simple.go

@@ -120,6 +120,11 @@ p: pvC0JBrEyUqtIIUvB2UUx/2a24c3Cvnu6AZ0D3IMBYAu...
 
 type benchSectorProvider map[storiface.SectorFileType]string
 
+func (b benchSectorProvider) AcquireSectorCopy(ctx context.Context, id storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
+	// there's no copying in this context
+	return b.AcquireSector(ctx, id, existing, allocate, ptype)
+}
+
 func (b benchSectorProvider) AcquireSector(ctx context.Context, id storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
 	out := storiface.SectorPaths{
 		ID:          id.ID,

cmd/lotus-miner/actor.go

@@ -21,7 +21,6 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/builtin"
-	minerV12 "github.com/filecoin-project/go-state-types/builtin/v12/miner"
 	"github.com/filecoin-project/go-state-types/builtin/v9/miner"
 	"github.com/filecoin-project/go-state-types/network"
 
@@ -50,7 +49,6 @@ var actorCmd = &cli.Command{
 		actorProposeChangeWorker,
 		actorConfirmChangeWorker,
 		actorCompactAllocatedCmd,
-		actorMovePartitionsCmd,
 		actorProposeChangeBeneficiary,
 		actorConfirmChangeBeneficiary,
 	},
@@ -1288,142 +1286,6 @@ var actorConfirmChangeBeneficiary = &cli.Command{
 	},
 }
 
-var actorMovePartitionsCmd = &cli.Command{
-	Name:  "move-partitions",
-	Usage: "move deadline of specified partitions from one to another",
-	Flags: []cli.Flag{
-		&cli.Int64SliceFlag{
-			Name:  "partition-indices",
-			Usage: "Indices of partitions to update, separated by comma",
-		},
-		&cli.Uint64Flag{
-			Name:  "orig-deadline",
-			Usage: "Deadline to move partition from",
-		},
-		&cli.Uint64Flag{
-			Name:  "dest-deadline",
-			Usage: "Deadline to move partition to",
-		},
-		&cli.BoolFlag{
-			Name:  "really-do-it",
-			Usage: "Actually send transaction performing the action",
-			Value: false,
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		if !cctx.Bool("really-do-it") {
-			fmt.Println("Pass --really-do-it to actually execute this action")
-			return nil
-		}
-
-		if cctx.Args().Present() {
-			return fmt.Errorf("please use flags to provide arguments")
-		}
-
-		ctx := lcli.ReqContext(cctx)
-
-		minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer closer()
-
-		maddr, err := minerApi.ActorAddress(ctx)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Miner: %s\n", color.BlueString("%s", maddr))
-
-		fullNodeApi, acloser, err := lcli.GetFullNodeAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer acloser()
-
-		minfo, err := fullNodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
-		if err != nil {
-			return err
-		}
-
-		origDeadline := cctx.Uint64("orig-deadline")
-		if origDeadline > miner.WPoStPeriodDeadlines {
-			return fmt.Errorf("orig-deadline %d out of range", origDeadline)
-		}
-		destDeadline := cctx.Uint64("dest-deadline")
-		if destDeadline > miner.WPoStPeriodDeadlines {
-			return fmt.Errorf("dest-deadline %d out of range", destDeadline)
-		}
-		if origDeadline == destDeadline {
-			return fmt.Errorf("dest-desdline cannot be the same as orig-deadline")
-		}
-
-		partitions := cctx.Int64Slice("partition-indices")
-		if len(partitions) == 0 {
-			return fmt.Errorf("must include at least one partition to move")
-		}
-
-		curPartitions, err := fullNodeApi.StateMinerPartitions(ctx, maddr, origDeadline, types.EmptyTSK)
-		if err != nil {
-			return fmt.Errorf("getting partitions for deadline %d: %w", origDeadline, err)
-		}
-		if len(partitions) > len(curPartitions) {
-			return fmt.Errorf("partition size(%d) cannot be bigger than current partition size(%d) for deadline %d", len(partitions), len(curPartitions), origDeadline)
-		}
-
-		fmt.Printf("Moving %d paritions\n", len(partitions))
-
-		partitionsBf := bitfield.New()
-		for _, partition := range partitions {
-			if partition >= int64(len(curPartitions)) {
-				return fmt.Errorf("partition index(%d) doesn't exist", partition)
-			}
-			partitionsBf.Set(uint64(partition))
-		}
-
-		params := minerV12.MovePartitionsParams{
-			OrigDeadline: origDeadline,
-			DestDeadline: destDeadline,
-			Partitions:   partitionsBf,
-		}
-
-		serializedParams, err := actors.SerializeParams(&params)
-		if err != nil {
-			return fmt.Errorf("serializing params: %w", err)
-		}
-
-		smsg, err := fullNodeApi.MpoolPushMessage(ctx, &types.Message{
-			From:   minfo.Worker,
-			To:     maddr,
-			Method: builtin.MethodsMiner.MovePartitions,
-			Value:  big.Zero(),
-			Params: serializedParams,
-		}, nil)
-		if err != nil {
-			return fmt.Errorf("mpool push: %w", err)
-		}
-
-		fmt.Println("MovePartitions Message CID:", smsg.Cid())
-
-		// wait for it to get mined into a block
-		fmt.Println("Waiting for block confirmation...")
-		wait, err := fullNodeApi.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
-		if err != nil {
-			return err
-		}
-
-		// check it executed successfully
-		if wait.Receipt.ExitCode.IsError() {
-			fmt.Println("Moving partitions failed!")
-			return err
-		}
-
-		fmt.Println("Move partition confirmed")
-
-		return nil
-	},
-}
-
 var actorCompactAllocatedCmd = &cli.Command{
 	Name:  "compact-allocated",
 	Usage: "compact allocated sectors bitfield",

cmd/lotus-shed/block.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var blockCmd = &cli.Command{
+	Name:      "block",
+	Usage:     "Output decoded block header in readeble form",
+	ArgsUsage: "[block header hex]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return lcli.IncorrectNumArgs(cctx)
+		}
+
+		b, err := hex.DecodeString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		var blk types.BlockHeader
+		if err := blk.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
+			return err
+		}
+
+		jb, err := json.MarshalIndent(blk, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(jb))
+		return nil
+	},
+}

cmd/lotus-shed/cron-count.go

@@ -1,14 +1,24 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
 
+	"github.com/ipfs/go-cid"
+	ipldcbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-state-types/abi"
+	miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner"
+	"github.com/filecoin-project/go-state-types/builtin/v11/util/adt"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 )
 
@@ -17,6 +27,245 @@ var cronWcCmd = &cli.Command{
 	Description: "cron stats",
 	Subcommands: []*cli.Command{
 		minerDeadlineCronCountCmd,
+		minerDeadlinePartitionMeasurementCmd,
+	},
+}
+
+type DeadlineRef struct {
+	To     string
+	Height abi.ChainEpoch
+	Gas    json.RawMessage
+}
+
+type DeadlineSummary struct {
+	Partitions      []PartitionSummary
+	PreCommitExpiry PreCommitExpiry
+	VestingDiff     VestingDiff
+}
+
+type PreCommitExpiry struct {
+	Expired []uint64
+}
+
+type VestingDiff struct {
+	PrevTableSize int
+	NewTableSize  int
+}
+
+type PartitionSummary struct {
+	Live   int
+	Dead   int
+	Faulty int
+	Diff   PartitionDiff
+}
+
+type PartitionDiff struct {
+	Faulted   int
+	Recovered int
+	Killed    int
+}
+
+var minerDeadlinePartitionMeasurementCmd = &cli.Command{
+	Name:        "deadline-summary",
+	Description: "",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "json",
+			Usage: "read input as json",
+			Value: true,
+		},
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset state to search on (pass comma separated array of cids)",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		// read in values to process
+		if !c.Bool("json") {
+			return xerrors.Errorf("unsupported non json input format")
+		}
+		var refStream []DeadlineRef
+		if err := json.NewDecoder(os.Stdin).Decode(&refStream); err != nil {
+			return xerrors.Errorf("failed to parse input: %w", err)
+		}
+
+		// go from height and sp addr to deadline partition data
+		n, acloser, err := lcli.GetFullNodeAPI(c)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+		ctx := lcli.ReqContext(c)
+
+		bs := ReadOnlyAPIBlockstore{n}
+		adtStore := adt.WrapStore(ctx, ipldcbor.NewCborStore(&bs))
+
+		dSummaries := make([]DeadlineSummary, len(refStream))
+		for j, ref := range refStream {
+			// get miner's deadline
+			tsBefore, err := n.ChainGetTipSetByHeight(ctx, ref.Height, types.EmptyTSK)
+			if err != nil {
+				return xerrors.Errorf("failed to get tipset at epoch: %d: %w", ref.Height, err)
+			}
+			tsAfter, err := n.ChainGetTipSetByHeight(ctx, ref.Height+1, types.EmptyTSK)
+			if err != nil {
+				return xerrors.Errorf("failed to get tipset at epoch %d: %w", ref.Height, err)
+			}
+			addr, err := address.NewFromString(ref.To)
+			if err != nil {
+				return xerrors.Errorf("faield to get address from input string: %w", err)
+			}
+			dline, err := n.StateMinerProvingDeadline(ctx, addr, tsBefore.Key())
+			if err != nil {
+				return xerrors.Errorf("failed to read proving deadline: %w", err)
+			}
+
+			// iterate through all partitions at epoch of processing
+			var pSummaries []PartitionSummary
+			psBefore, err := n.StateMinerPartitions(ctx, addr, dline.Index, tsBefore.Key())
+			if err != nil {
+				return xerrors.Errorf("failed to get partitions: %w", err)
+			}
+			psAfter, err := n.StateMinerPartitions(ctx, addr, dline.Index, tsAfter.Key())
+			if err != nil {
+				return xerrors.Errorf("failed to get partitions: %w", err)
+			}
+			if len(psBefore) != len(psAfter) {
+				return xerrors.Errorf("faield")
+			}
+
+			type partitionCount struct {
+				live       int
+				dead       int
+				faulty     int
+				recovering int
+			}
+			countPartition := func(p api.Partition) (partitionCount, error) {
+				liveSectors, err := p.LiveSectors.All(abi.MaxSectorNumber)
+				if err != nil {
+					return partitionCount{}, xerrors.Errorf("failed to count live sectors in partition: %w", err)
+				}
+				allSectors, err := p.AllSectors.All(abi.MaxSectorNumber)
+				if err != nil {
+					return partitionCount{}, xerrors.Errorf("failed to count all sectors in partition: %w", err)
+				}
+				faultySectors, err := p.FaultySectors.All(abi.MaxSectorNumber)
+				if err != nil {
+					return partitionCount{}, xerrors.Errorf("failed to count faulty sectors in partition: %w", err)
+				}
+				recoveringSectors, err := p.RecoveringSectors.All(abi.MaxSectorNumber)
+				if err != nil {
+					return partitionCount{}, xerrors.Errorf("failed to count recovering sectors in partition: %w", err)
+				}
+
+				return partitionCount{
+					live:       len(liveSectors),
+					dead:       len(allSectors) - len(liveSectors),
+					faulty:     len(faultySectors),
+					recovering: len(recoveringSectors),
+				}, nil
+			}
+
+			countVestingTable := func(table cid.Cid) (int, error) {
+				var vestingTable miner11.VestingFunds
+				if err := adtStore.Get(ctx, table, &vestingTable); err != nil {
+					return 0, err
+				}
+				return len(vestingTable.Funds), nil
+			}
+
+			for i := 0; i < len(psBefore); i++ {
+				cntBefore, err := countPartition(psBefore[i])
+				if err != nil {
+					return err
+				}
+				cntAfter, err := countPartition(psAfter[i])
+				if err != nil {
+					return err
+				}
+				pSummaries = append(pSummaries, PartitionSummary{
+					Live:   cntBefore.live,
+					Dead:   cntBefore.dead,
+					Faulty: cntBefore.faulty,
+					Diff: PartitionDiff{
+						Faulted:   cntAfter.faulty - cntBefore.faulty,
+						Recovered: cntBefore.recovering - cntAfter.recovering,
+						Killed:    cntAfter.dead - cntBefore.dead,
+					},
+				})
+			}
+
+			// Precommit and vesting table data
+			// Before
+			aBefore, err := n.StateGetActor(ctx, addr, tsBefore.Key())
+			if err != nil {
+				return err
+			}
+			var st miner11.State
+			err = adtStore.Get(ctx, aBefore.Head, &st)
+			if err != nil {
+				return err
+			}
+			expiryQArray, err := adt.AsArray(adtStore, st.PreCommittedSectorsCleanUp, miner11.PrecommitCleanUpAmtBitwidth)
+			if err != nil {
+				return err
+			}
+			var sectorsBf bitfield.BitField
+			var accumulator []uint64
+			h := ref.Height
+			if err := expiryQArray.ForEach(&sectorsBf, func(i int64) error {
+				if abi.ChainEpoch(i) > h {
+					return nil
+				}
+				sns, err := sectorsBf.All(abi.MaxSectorNumber)
+				if err != nil {
+					return err
+				}
+				accumulator = append(accumulator, sns...)
+				return nil
+			}); err != nil {
+				return err
+			}
+
+			vestingBefore, err := countVestingTable(st.VestingFunds)
+			if err != nil {
+				return err
+			}
+
+			// After
+			aAfter, err := n.StateGetActor(ctx, addr, tsAfter.Key())
+			if err != nil {
+				return err
+			}
+			var stAfter miner11.State
+			err = adtStore.Get(ctx, aAfter.Head, &stAfter)
+			if err != nil {
+				return err
+			}
+
+			vestingAfter, err := countVestingTable(stAfter.VestingFunds)
+			if err != nil {
+				return err
+			}
+
+			dSummaries[j] = DeadlineSummary{
+				Partitions: pSummaries,
+				PreCommitExpiry: PreCommitExpiry{
+					Expired: accumulator,
+				},
+				VestingDiff: VestingDiff{
+					PrevTableSize: vestingBefore,
+					NewTableSize:  vestingAfter,
+				},
+			}
+
+		}
+
+		// output partition info
+		if err := json.NewEncoder(os.Stdout).Encode(dSummaries); err != nil {
+			return err
+		}
+		return nil
 	},
 }
 

cmd/lotus-shed/election.go

@@ -219,7 +219,7 @@ func backTestWinner(ctx context.Context, miner address.Address, round abi.ChainE
 		brand = bvals[len(bvals)-1]
 	}
 
-	winner, err := gen.IsRoundWinner(ctx, ts, round, miner, brand, mbi, api)
+	winner, err := gen.IsRoundWinner(ctx, round, miner, brand, mbi, api)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to check if we win next round: %w", err)
 	}

cmd/lotus-shed/gas-estimation.go

@@ -16,7 +16,6 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/beacon"
 	"github.com/filecoin-project/lotus/chain/beacon/drand"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
@@ -100,15 +99,11 @@ var gasTraceCmd = &cli.Command{
 			return err
 		}
 
-		dcs := build.DrandConfigSchedule()
-		shd := beacon.Schedule{}
-		for _, dc := range dcs {
-			bc, err := drand.NewDrandBeacon(MAINNET_GENESIS_TIME, build.BlockDelaySecs, nil, dc.Config)
-			if err != nil {
-				return xerrors.Errorf("creating drand beacon: %w", err)
-			}
-			shd = append(shd, beacon.BeaconPoint{Start: dc.Start, Beacon: bc})
+		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), MAINNET_GENESIS_TIME, nil)
+		if err != nil {
+			return err
 		}
+
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 		defer cs.Close() //nolint:errcheck
 
@@ -200,14 +195,9 @@ var replayOfflineCmd = &cli.Command{
 			return err
 		}
 
-		dcs := build.DrandConfigSchedule()
-		shd := beacon.Schedule{}
-		for _, dc := range dcs {
-			bc, err := drand.NewDrandBeacon(MAINNET_GENESIS_TIME, build.BlockDelaySecs, nil, dc.Config)
-			if err != nil {
-				return xerrors.Errorf("creating drand beacon: %w", err)
-			}
-			shd = append(shd, beacon.BeaconPoint{Start: dc.Start, Beacon: bc})
+		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), MAINNET_GENESIS_TIME, nil)
+		if err != nil {
+			return err
 		}
 
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)

cmd/lotus-shed/indexes.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"path"
@@ -8,12 +9,18 @@ import (
 	"strings"
 
 	"github.com/mitchellh/go-homedir"
+	"github.com/multiformats/go-varint"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	builtintypes "github.com/filecoin-project/go-state-types/builtin"
 	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/exitcode"
 
+	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 	lcli "github.com/filecoin-project/lotus/cli"
 )
@@ -31,6 +38,291 @@ var indexesCmd = &cli.Command{
 		withCategory("msgindex", backfillMsgIndexCmd),
 		withCategory("msgindex", pruneMsgIndexCmd),
 		withCategory("txhash", backfillTxHashCmd),
+		withCategory("events", backfillEventsCmd),
+	},
+}
+
+var backfillEventsCmd = &cli.Command{
+	Name:  "backfill-events",
+	Usage: "Backfill the events.db for a number of epochs starting from a specified height",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "from",
+			Value: 0,
+			Usage: "the tipset height to start backfilling from (0 is head of chain)",
+		},
+		&cli.IntFlag{
+			Name:  "epochs",
+			Value: 2000,
+			Usage: "the number of epochs to backfill",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		srv, err := lcli.GetFullNodeServices(cctx)
+		if err != nil {
+			return err
+		}
+		defer srv.Close() //nolint:errcheck
+
+		api := srv.FullNodeAPI()
+		ctx := lcli.ReqContext(cctx)
+
+		// currTs will be the tipset where we start backfilling from
+		currTs, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+		if cctx.IsSet("from") {
+			// we need to fetch the tipset after the epoch being specified since we will need to advance currTs
+			currTs, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(cctx.Int("from")+1), currTs.Key())
+			if err != nil {
+				return err
+			}
+		}
+
+		// advance currTs by one epoch and maintain prevTs as the previous tipset (this allows us to easily use the ChainGetParentMessages/Receipt API)
+		prevTs := currTs
+		currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
+		if err != nil {
+			return fmt.Errorf("failed to load tipset %s: %w", prevTs.Parents(), err)
+		}
+
+		epochs := cctx.Int("epochs")
+
+		basePath, err := homedir.Expand(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		dbPath := path.Join(basePath, "sqlite", "events.db")
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err := db.Close()
+			if err != nil {
+				fmt.Printf("ERROR: closing db: %s", err)
+			}
+		}()
+
+		addressLookups := make(map[abi.ActorID]address.Address)
+
+		resolveFn := func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) {
+			// we only want to match using f4 addresses
+			idAddr, err := address.NewIDAddress(uint64(emitter))
+			if err != nil {
+				return address.Undef, false
+			}
+
+			actor, err := api.StateGetActor(ctx, idAddr, ts.Key())
+			if err != nil || actor.Address == nil {
+				return address.Undef, false
+			}
+
+			// if robust address is not f4 then we won't match against it so bail early
+			if actor.Address.Protocol() != address.Delegated {
+				return address.Undef, false
+			}
+
+			// we have an f4 address, make sure it's assigned by the EAM
+			if namespace, _, err := varint.FromUvarint(actor.Address.Payload()); err != nil || namespace != builtintypes.EthereumAddressManagerActorID {
+				return address.Undef, false
+			}
+			return *actor.Address, true
+		}
+
+		isIndexedValue := func(b uint8) bool {
+			// currently we mark the full entry as indexed if either the key
+			// or the value are indexed; in the future we will need finer-grained
+			// management of indices
+			return b&(types.EventFlagIndexedKey|types.EventFlagIndexedValue) > 0
+		}
+
+		var totalEventsAffected int64
+		var totalEntriesAffected int64
+
+		processHeight := func(ctx context.Context, cnt int, msgs []lapi.Message, receipts []*types.MessageReceipt) error {
+			tx, err := db.BeginTx(ctx, nil)
+			if err != nil {
+				return fmt.Errorf("failed to start transaction: %w", err)
+			}
+			defer tx.Rollback() //nolint:errcheck
+
+			stmtSelectEvent, err := tx.Prepare("SELECT MAX(id) from event WHERE height=? AND tipset_key=? and tipset_key_cid=? and emitter_addr=? and event_index=? and message_cid=? and message_index=? and reverted=false")
+			if err != nil {
+				return err
+			}
+			stmtEvent, err := tx.Prepare("INSERT INTO event (height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted) VALUES(?, ?, ?, ?, ?, ?, ?, ?)")
+			if err != nil {
+				return err
+			}
+			stmtEntry, err := tx.Prepare("INSERT INTO event_entry(event_id, indexed, flags, key, codec, value) VALUES(?, ?, ?, ?, ?, ?)")
+			if err != nil {
+				return err
+			}
+
+			var eventsAffected int64
+			var entriesAffected int64
+
+			// loop over each message receipt and backfill the events
+			for idx, receipt := range receipts {
+				msg := msgs[idx]
+
+				if receipt.ExitCode != exitcode.Ok {
+					continue
+				}
+
+				if receipt.EventsRoot == nil {
+					continue
+				}
+
+				events, err := api.ChainGetEvents(ctx, *receipt.EventsRoot)
+				if err != nil {
+					return fmt.Errorf("failed to load events for tipset %s: %w", currTs, err)
+				}
+
+				for eventIdx, event := range events {
+					addr, found := addressLookups[event.Emitter]
+					if !found {
+						var ok bool
+						addr, ok = resolveFn(ctx, event.Emitter, currTs)
+						if !ok {
+							// not an address we will be able to match against
+							continue
+						}
+						addressLookups[event.Emitter] = addr
+					}
+
+					tsKeyCid, err := currTs.Key().Cid()
+					if err != nil {
+						return fmt.Errorf("failed to get tipset key cid: %w", err)
+					}
+
+					// select the highest event id that exists in database, or null if none exists
+					var entryID sql.NullInt64
+					err = stmtSelectEvent.QueryRow(
+						currTs.Height(),
+						currTs.Key().Bytes(),
+						tsKeyCid.Bytes(),
+						addr.Bytes(),
+						eventIdx,
+						msg.Cid.Bytes(),
+						idx,
+					).Scan(&entryID)
+					if err != nil {
+						return fmt.Errorf("error checking if event exists: %w", err)
+					}
+
+					// we already have this event
+					if entryID.Valid {
+						continue
+					}
+
+					// event does not exist, lets backfill it
+					res, err := tx.Stmt(stmtEvent).Exec(
+						currTs.Height(),      // height
+						currTs.Key().Bytes(), // tipset_key
+						tsKeyCid.Bytes(),     // tipset_key_cid
+						addr.Bytes(),         // emitter_addr
+						eventIdx,             // event_index
+						msg.Cid.Bytes(),      // message_cid
+						idx,                  // message_index
+						false,                // reverted
+					)
+					if err != nil {
+						return fmt.Errorf("error inserting event: %w", err)
+					}
+
+					entryID.Int64, err = res.LastInsertId()
+					if err != nil {
+						return fmt.Errorf("could not get last insert id: %w", err)
+					}
+
+					rowsAffected, err := res.RowsAffected()
+					if err != nil {
+						return fmt.Errorf("could not get rows affected: %w", err)
+					}
+					eventsAffected += rowsAffected
+
+					// backfill the event entries
+					for _, entry := range event.Entries {
+						_, err := tx.Stmt(stmtEntry).Exec(
+							entryID.Int64,               // event_id
+							isIndexedValue(entry.Flags), // indexed
+							[]byte{entry.Flags},         // flags
+							entry.Key,                   // key
+							entry.Codec,                 // codec
+							entry.Value,                 // value
+						)
+						if err != nil {
+							return fmt.Errorf("error inserting entry: %w", err)
+						}
+
+						rowsAffected, err := res.RowsAffected()
+						if err != nil {
+							return fmt.Errorf("could not get rows affected: %w", err)
+						}
+						entriesAffected += rowsAffected
+					}
+				}
+			}
+
+			err = tx.Commit()
+			if err != nil {
+				return fmt.Errorf("failed to commit transaction: %w", err)
+			}
+
+			log.Infof("[%d] backfilling actor events epoch:%d, eventsAffected:%d, entriesAffected:%d", cnt, currTs.Height(), eventsAffected, entriesAffected)
+
+			totalEventsAffected += eventsAffected
+			totalEntriesAffected += entriesAffected
+
+			return nil
+		}
+
+		for i := 0; i < epochs; i++ {
+			select {
+			case <-ctx.Done():
+				return nil
+			default:
+			}
+
+			blockCid := prevTs.Blocks()[0].Cid()
+
+			// get messages for the parent of the previous tipset (which will be currTs)
+			msgs, err := api.ChainGetParentMessages(ctx, blockCid)
+			if err != nil {
+				return fmt.Errorf("failed to get parent messages for block %s: %w", blockCid, err)
+			}
+
+			// get receipts for the parent of the previous tipset (which will be currTs)
+			receipts, err := api.ChainGetParentReceipts(ctx, blockCid)
+			if err != nil {
+				return fmt.Errorf("failed to get parent receipts for block %s: %w", blockCid, err)
+			}
+
+			if len(msgs) != len(receipts) {
+				return fmt.Errorf("mismatched in message and receipt count: %d != %d", len(msgs), len(receipts))
+			}
+
+			err = processHeight(ctx, i, msgs, receipts)
+			if err != nil {
+				return err
+			}
+
+			// advance prevTs and currTs up the chain
+			prevTs = currTs
+			currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
+			if err != nil {
+				return fmt.Errorf("failed to load tipset %s: %w", currTs, err)
+			}
+		}
+
+		log.Infof("backfilling events complete, totalEventsAffected:%d, totalEntriesAffected:%d", totalEventsAffected, totalEntriesAffected)
+
+		return nil
 	},
 }
 

cmd/lotus-shed/invariants.go

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/go-state-types/builtin"
 	v10 "github.com/filecoin-project/go-state-types/builtin/v10"
 	v11 "github.com/filecoin-project/go-state-types/builtin/v11"
+	v12 "github.com/filecoin-project/go-state-types/builtin/v12"
 	v8 "github.com/filecoin-project/go-state-types/builtin/v8"
 	v9 "github.com/filecoin-project/go-state-types/builtin/v9"
 
@@ -149,6 +150,13 @@ var invariantsCmd = &cli.Command{
 			if err != nil {
 				return xerrors.Errorf("checking state invariants: %w", err)
 			}
+		case actorstypes.Version12:
+			messages, err = v12.CheckStateInvariants(actorTree, abi.ChainEpoch(epoch), actorCodeCids)
+			if err != nil {
+				return xerrors.Errorf("checking state invariants: %w", err)
+			}
+		default:
+			return xerrors.Errorf("unsupported actor version: %v", av)
 		}
 
 		fmt.Println("completed, took ", time.Since(startTime))

cmd/lotus-shed/main.go

@@ -24,6 +24,7 @@ func main() {
 	local := []*cli.Command{
 		addressCmd,
 		statActorCmd,
+		statSnapshotCmd,
 		statObjCmd,
 		base64Cmd,
 		base32Cmd,
@@ -90,6 +91,7 @@ func main() {
 		indexesCmd,
 		FevmAnalyticsCmd,
 		mismatchesCmd,
+		blockCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/msg.go

@@ -26,6 +26,12 @@ var msgCmd = &cli.Command{
 	Aliases:   []string{"msg"},
 	Usage:     "Translate message between various formats",
 	ArgsUsage: "Message in any form",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "exec-trace",
+			Usage: "Print the execution trace",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.NArg() != 1 {
 			return lcli.IncorrectNumArgs(cctx)
@@ -36,6 +42,48 @@ var msgCmd = &cli.Command{
 			return err
 		}
 
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		// Get the CID of the message
+		mcid := msg.Cid()
+
+		// Search for the message on-chain
+		lookup, err := api.StateSearchMsg(ctx, mcid)
+		if err != nil {
+			return err
+		}
+		if lookup == nil {
+			fmt.Println("Message not found on-chain. Continuing...")
+		} else {
+			// Replay the message to get the execution trace
+			res, err := api.StateReplay(ctx, types.EmptyTSK, mcid)
+			if err != nil {
+				return xerrors.Errorf("replay call failed: %w", err)
+			}
+
+			if cctx.Bool("exec-trace") {
+				// Print the execution trace
+				color.Green("Execution trace:")
+				trace, err := json.MarshalIndent(res.ExecutionTrace, "", "  ")
+				if err != nil {
+					return xerrors.Errorf("marshaling execution trace: %w", err)
+				}
+				fmt.Println(string(trace))
+				fmt.Println()
+
+				color.Green("Receipt:")
+				fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode)
+				fmt.Printf("Return: %x\n", res.MsgRct.Return)
+				fmt.Printf("Gas Used: %d\n", res.MsgRct.GasUsed)
+			}
+		}
+
 		switch msg := msg.(type) {
 		case *types.SignedMessage:
 			return printSignedMessage(cctx, msg)

cmd/lotus-shed/state-stats.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
+	"path"
 	"reflect"
+	"sort"
 	"sync"
 
 	"github.com/docker/go-units"
@@ -21,8 +24,12 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	gstactors "github.com/filecoin-project/go-state-types/actors"
+	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
@@ -49,6 +56,19 @@ type fieldItem struct {
 	Stats api.ObjStat
 }
 
+type job struct {
+	c   cid.Cid
+	key string // prefix path for the region being recorded i.e. "/state/mineractor"
+}
+type cidCall struct {
+	c    cid.Cid
+	resp chan bool
+}
+type result struct {
+	key   string
+	stats api.ObjStat
+}
+
 type cacheNodeGetter struct {
 	ds    format.NodeGetter
 	cache *lru.TwoQueueCache[cid.Cid, format.Node]
@@ -166,39 +186,13 @@ var statObjCmd = &cli.Command{
 			return err
 		}
 
-		r, err := repo.NewFS(cctx.String("repo"))
-		if err != nil {
-			return xerrors.Errorf("opening fs repo: %w", err)
-		}
-
-		exists, err := r.Exists()
+		h, err := loadChainStore(ctx, cctx.String("repo"))
 		if err != nil {
 			return err
 		}
-		if !exists {
-			return xerrors.Errorf("lotus repo doesn't exist")
-		}
+		defer h.closer()
 
-		lr, err := r.Lock(repo.FullNode)
-		if err != nil {
-			return err
-		}
-		defer lr.Close() //nolint:errcheck
-
-		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
-		if err != nil {
-			return fmt.Errorf("failed to open blockstore: %w", err)
-		}
-
-		defer func() {
-			if c, ok := bs.(io.Closer); ok {
-				if err := c.Close(); err != nil {
-					log.Warnf("failed to close blockstore: %s", err)
-				}
-			}
-		}()
-
-		dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+		dag := merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs)))
 		dsc := &dagStatCollector{
 			ds:   dag,
 			walk: carWalkFunc,
@@ -212,6 +206,376 @@ var statObjCmd = &cli.Command{
 	},
 }
 
+type StoreHandle struct {
+	bs     blockstore.Blockstore
+	cs     *store.ChainStore
+	sm     *stmgr.StateManager
+	closer func()
+}
+
+func loadChainStore(ctx context.Context, repoPath string) (*StoreHandle, error) {
+	r, err := repo.NewFS(repoPath)
+	if err != nil {
+		return nil, xerrors.Errorf("opening fs repo: %w", err)
+	}
+
+	exists, err := r.Exists()
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, xerrors.Errorf("lotus repo doesn't exist")
+	}
+
+	lr, err := r.Lock(repo.FullNode)
+	if err != nil {
+		return nil, err
+	}
+
+	bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open blockstore: %w", err)
+	}
+
+	closer := func() {
+		if err := lr.Close(); err != nil {
+			log.Warnf("failed to close locked repo: %s", err)
+		}
+		if c, ok := bs.(io.Closer); ok {
+			if err := c.Close(); err != nil {
+				log.Warnf("failed to close blockstore: %s", err)
+			}
+		}
+	}
+
+	mds, err := lr.Datastore(context.Background(), "/metadata")
+	if err != nil {
+		return nil, err
+	}
+
+	cs := store.NewChainStore(bs, bs, mds, nil, nil)
+	if err := cs.Load(ctx); err != nil {
+		return nil, fmt.Errorf("failed to load chain store: %w", err)
+	}
+
+	tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc)
+	sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open state manager: %w", err)
+	}
+	handle := StoreHandle{
+		bs:     bs,
+		sm:     sm,
+		cs:     cs,
+		closer: closer,
+	}
+
+	return &handle, nil
+}
+
+func pipeline(ctx context.Context, name string, numWorkers int, createJobs func(ctx context.Context, jobCh chan job, resultCh chan result) error,
+	worker func(ctx context.Context, id int, jobCh chan job, resultCh chan result) error,
+	processResults func(ctx context.Context, resultCh chan result) error) error {
+
+	eg, egctx := errgroup.WithContext(ctx)
+	jobCh := make(chan job, numWorkers)
+	resultCh := make(chan result)
+	var resultWriterWg sync.WaitGroup
+
+	resultWriterWg.Add(1)
+	eg.Go(func() error {
+		defer resultWriterWg.Done()
+		defer func() {
+			close(jobCh)
+		}()
+		return createJobs(ctx, jobCh, resultCh)
+	})
+
+	var id int
+	for w := 0; w < numWorkers; w++ {
+		id = w
+
+		resultWriterWg.Add(1)
+		eg.Go(func() error {
+			defer resultWriterWg.Done()
+			return worker(egctx, id, jobCh, resultCh)
+		})
+	}
+
+	eg.Go(func() error {
+		return processResults(ctx, resultCh)
+	})
+
+	// close result channel when workers are done sending to it.
+	eg.Go(func() error {
+		resultWriterWg.Wait()
+		close(resultCh)
+		return nil
+	})
+
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("failed pipeline %s: %w", name, err)
+	}
+	return nil
+}
+
+var statSnapshotCmd = &cli.Command{
+	Name:  "stat-snapshot",
+	Usage: "calculates the space usage of a snapshot taken from the given tipset",
+	Description: `Walk the chain back to lightweight snapshot size and break down space usage into high level 
+	categories: headers, messages, receipts, latest state root, and churn from earlier state roots.
+	State root and churn space is further broken down by actor type and immediate top level fields  
+	`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset to call method on (pass comma separated array of cids)",
+		},
+		&cli.IntFlag{
+			Name:  "workers",
+			Usage: "number of workers to use when processing",
+			Value: 10,
+		},
+		&cli.IntFlag{
+			Name:  "dag-cache-size",
+			Usage: "cache size per worker (setting to 0 disables)",
+			Value: 8092,
+		},
+		&cli.BoolFlag{
+			Name:  "pretty",
+			Usage: "print formated output instead of ldjson",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		h, err := loadChainStore(ctx, cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+		defer h.closer()
+		tsr := &ChainStoreTipSetResolver{
+			Chain: h.cs,
+		}
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, tsr)
+		if err != nil {
+			return err
+		}
+
+		numWorkers := cctx.Int("workers")
+		dagCacheSize := cctx.Int("dag-cache-size")
+
+		cidCh := make(chan cidCall, numWorkers)
+		summary := make(map[string]api.ObjStat)
+		// snapshot root objects with no additional bytes or links
+		summary["/"] = api.ObjStat{Size: 0, Links: 0}
+		summary["/statetree"] = api.ObjStat{Size: 0, Links: 0}
+
+		combine := func(statsA, statsB api.ObjStat) api.ObjStat {
+			return api.ObjStat{
+				Size:  statsA.Size + statsB.Size,
+				Links: statsA.Links + statsB.Links,
+			}
+		}
+
+		// Threadsafe cid set lives across different pipelines so not part of error group
+		go func() {
+			seen := cid.NewSet()
+			for {
+				select {
+				case call := <-cidCh:
+					call.resp <- seen.Visit(call.c)
+				case <-ctx.Done():
+					log.Infof("shutting down cid set goroutine: %s", ctx.Err())
+					return
+				}
+			}
+		}()
+		visit := func(c cid.Cid) bool {
+			ch := make(chan bool)
+			cidCh <- cidCall{c: c, resp: ch}
+			out := <-ch
+			return out
+		}
+		// Stage 1 walk all actors in latest state root
+		createJobsStage1 := func(ctx context.Context, jobCh chan job, _ chan result) error {
+			st, err := h.sm.StateTree(ts.ParentState())
+			if err != nil {
+				return err
+			}
+
+			return st.ForEach(func(_ address.Address, act *types.Actor) error {
+				actType := builtin.ActorNameByCode(act.Code)
+				actType = path.Base(actType) // strip away fil/<nv>
+				if actType == "<unknown>" {
+					actType = act.Code.String()
+				}
+				jobCh <- job{c: act.Head, key: fmt.Sprintf("/statetree/latest/%s", actType)}
+
+				return nil
+			})
+		}
+
+		worker := func(ctx context.Context, id int, jobCh chan job, resultCh chan result) error {
+			var dag format.NodeGetter = merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs)))
+			if dagCacheSize != 0 {
+				var err error
+				dag, err = newCacheNodeGetter(merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs))), dagCacheSize)
+				if err != nil {
+					return err
+				}
+			}
+
+			for job := range jobCh {
+				stats, err := collectSnapshotJobStats(ctx, job, dag, visit)
+				if err != nil {
+					return err
+				}
+				for _, stat := range stats {
+					select {
+					case resultCh <- stat:
+					case <-ctx.Done():
+						return ctx.Err()
+					}
+				}
+			}
+			return nil
+		}
+
+		processResults := func(ctx context.Context, resultCh chan result) error {
+			for result := range resultCh {
+				if stat, ok := summary[result.key]; ok {
+					summary[result.key] = combine(stat, result.stats)
+
+				} else {
+					summary[result.key] = result.stats
+				}
+			}
+			return nil
+		}
+
+		if err := pipeline(ctx, "Latest State Actors", numWorkers, createJobsStage1, worker, processResults); err != nil {
+			return err
+		}
+
+		// Stage 2: walk the top of the latest state root
+		createJobsStage2 := func(ctx context.Context, jobCh chan job, _ chan result) error {
+			jobCh <- job{c: ts.ParentState(), key: "/statetree/latest"}
+			return nil
+		}
+
+		if err := pipeline(ctx, "Latest State HAMT", numWorkers, createJobsStage2, worker, processResults); err != nil {
+			return err
+		}
+
+		// Stage 3 walk the rest of the chain: headers, messages, churn
+		// ordering:
+		// for each header send jobs for messages, receipts, state tree churn
+		//    don't walk header directly as it would just walk everything including parent tipsets
+
+		churnStateRoots := cid.NewSet()
+		createJobsStage3 := func(ctx context.Context, jobCh chan job, resultCh chan result) error {
+			// walk chain
+			var dag format.NodeGetter = merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs)))
+			if dagCacheSize != 0 {
+				var err error
+				dag, err = newCacheNodeGetter(merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs))), dagCacheSize)
+				if err != nil {
+					return err
+				}
+			}
+
+			blocksToWalk := ts.Cids()
+			startHeight := ts.Height()
+			snapshotStateLimit := abi.ChainEpoch(2000)
+
+			churnActorCache := cid.NewSet()
+			blocksTracked := cid.NewSet()
+			for len(blocksToWalk) > 0 {
+				blkCid := blocksToWalk[0]
+				blocksToWalk = blocksToWalk[1:]
+				nd, err := dag.Get(ctx, blkCid)
+				if err != nil {
+					return xerrors.Errorf("getting block: %w", err)
+				}
+
+				var b types.BlockHeader
+				if err := b.UnmarshalCBOR(bytes.NewBuffer(nd.RawData())); err != nil {
+					return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blkCid, err)
+				}
+
+				// header directly to result channel
+				resultCh <- result{key: "/headers", stats: api.ObjStat{Size: uint64(len(nd.RawData())), Links: uint64(len(nd.Links()))}}
+				// message job
+				if b.Height > startHeight-snapshotStateLimit {
+					jobCh <- job{key: "/messages", c: b.Messages}
+				}
+
+				// state churn job
+				if b.Height > startHeight-snapshotStateLimit {
+					if churnStateRoots.Visit(b.ParentStateRoot) {
+						st, err := h.sm.StateTree(b.ParentStateRoot)
+						if err != nil {
+							return err
+						}
+
+						err = st.ForEach(func(_ address.Address, act *types.Actor) error {
+							if churnActorCache.Visit(act.Head) {
+								actType := builtin.ActorNameByCode(act.Code)
+								actType = path.Base(actType) // strip away fil/<nv>
+								if actType == "<unknown>" {
+									actType = act.Code.String()
+								}
+								jobCh <- job{c: act.Head, key: fmt.Sprintf("/statetree/churn/%s", actType)}
+							}
+
+							return nil
+						})
+						if err != nil {
+							return err
+						}
+					}
+				}
+				for _, blkCid := range b.Parents {
+					if blocksTracked.Visit(blkCid) && b.Height != 0 {
+						blocksToWalk = append(blocksToWalk, blkCid)
+					}
+				}
+			}
+			return nil
+		}
+
+		if err := pipeline(ctx, "Churn, Headers, Messages", numWorkers, createJobsStage3, worker, processResults); err != nil {
+			return err
+		}
+
+		// step 1 clean things up and get a nice abstraction to reuse
+		// Stage 4 walk all actor HAMTs for churn
+
+		createJobsStage4 := func(ctx context.Context, jobCh chan job, _ chan result) error {
+			return churnStateRoots.ForEach(func(c cid.Cid) error {
+				jobCh <- job{c: c, key: "/statetree/churn"}
+				return nil
+			})
+		}
+
+		if err := pipeline(ctx, "Churn HAMT", numWorkers, createJobsStage4, worker, processResults); err != nil {
+			return err
+		}
+
+		if cctx.Bool("pretty") {
+			DumpSnapshotStats(summary)
+		} else {
+			if err := DumpJSON(summary); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
 var statActorCmd = &cli.Command{
 	Name:  "stat-actor",
 	Usage: "calculates the size of actors and their immeidate structures",
@@ -265,57 +629,14 @@ to reduce the number of decode operations performed by caching the decoded objec
 				addrs = append(addrs, addr)
 			}
 		}
-
-		r, err := repo.NewFS(cctx.String("repo"))
-		if err != nil {
-			return xerrors.Errorf("opening fs repo: %w", err)
-		}
-
-		exists, err := r.Exists()
-		if err != nil {
-			return err
-		}
-		if !exists {
-			return xerrors.Errorf("lotus repo doesn't exist")
-		}
-
-		lr, err := r.Lock(repo.FullNode)
-		if err != nil {
-			return err
-		}
-		defer lr.Close() //nolint:errcheck
-
-		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
-		if err != nil {
-			return fmt.Errorf("failed to open blockstore: %w", err)
-		}
-
-		defer func() {
-			if c, ok := bs.(io.Closer); ok {
-				if err := c.Close(); err != nil {
-					log.Warnf("failed to close blockstore: %s", err)
-				}
-			}
-		}()
-
-		mds, err := lr.Datastore(context.Background(), "/metadata")
-		if err != nil {
-			return err
-		}
-
-		cs := store.NewChainStore(bs, bs, mds, nil, nil)
-		if err := cs.Load(ctx); err != nil {
-			return nil
-		}
-
-		tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc)
-		sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
+		h, err := loadChainStore(ctx, cctx.String("repo"))
 		if err != nil {
 			return err
 		}
+		defer h.closer()
 
 		tsr := &ChainStoreTipSetResolver{
-			Chain: cs,
+			Chain: h.cs,
 		}
 
 		ts, err := lcli.LoadTipSet(ctx, cctx, tsr)
@@ -327,7 +648,7 @@ to reduce the number of decode operations performed by caching the decoded objec
 
 		if len(addrs) == 0 && cctx.Bool("all") {
 			var err error
-			addrs, err = sm.ListAllActors(ctx, ts)
+			addrs, err = h.sm.ListAllActors(ctx, ts)
 			if err != nil {
 				return err
 			}
@@ -354,15 +675,15 @@ to reduce the number of decode operations performed by caching the decoded objec
 						return nil
 					}
 
-					actor, err := sm.LoadActor(ctx, addr, ts)
+					actor, err := h.sm.LoadActor(ctx, addr, ts)
 					if err != nil {
 						return err
 					}
 
-					var dag format.NodeGetter = merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+					var dag format.NodeGetter = merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs)))
 					if dagCacheSize != 0 {
 						var err error
-						dag, err = newCacheNodeGetter(merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))), dagCacheSize)
+						dag, err = newCacheNodeGetter(merkledag.NewDAGService(blockservice.New(h.bs, offline.Exchange(h.bs))), dagCacheSize)
 						if err != nil {
 							return err
 						}
@@ -427,6 +748,93 @@ to reduce the number of decode operations performed by caching the decoded objec
 	},
 }
 
+func collectSnapshotJobStats(ctx context.Context, in job, dag format.NodeGetter, visit func(c cid.Cid) bool) ([]result, error) {
+	// "state" and "churn" attempt further breakdown by actor type
+	if !(path.Dir(in.key) == "/statetree/latest") && !(path.Dir(in.key) == "/statetree/churn") {
+		dsc := &dagStatCollector{
+			ds:   dag,
+			walk: carWalkFunc,
+		}
+
+		if err := merkledag.Walk(ctx, dsc.walkLinks, in.c, visit, merkledag.Concurrent()); err != nil {
+			return nil, err
+		}
+		return []result{{key: in.key, stats: dsc.stats}}, nil
+	}
+
+	// in.c is an actor head cid, try to unmarshal and create sub keys for different regions of state
+	nd, err := dag.Get(ctx, in.c)
+	if err != nil {
+		return nil, err
+	}
+	subjobs := make([]job, 0)
+	results := make([]result, 0)
+
+	// reconstruct actor for state parsing from key
+	av, err := gstactors.VersionForNetwork(network.Version20)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get actors version for network: %w", err)
+	}
+	code, ok := actors.GetActorCodeID(av, path.Base(in.key))
+	if !ok { // try parsing key directly
+		code, err = cid.Parse(path.Base(in.key))
+		if err != nil {
+			log.Debugf("failing to parse actor string: %s", path.Base(in.key))
+		}
+	}
+
+	actor := types.ActorV5{Head: in.c, Code: code}
+	oif, err := vm.DumpActorState(consensus.NewTipSetExecutor(filcns.RewardFunc).NewActorRegistry(), &actor, nd.RawData())
+	if err != nil {
+		oif = nil
+	}
+	// Account actors return nil from DumpActorState as they have no state
+	if oif != nil {
+		v := reflect.Indirect(reflect.ValueOf(oif))
+		for i := 0; i < v.NumField(); i++ {
+			varName := v.Type().Field(i).Name
+			varType := v.Type().Field(i).Type
+			varValue := v.Field(i).Interface()
+
+			if varType == reflect.TypeOf(cid.Cid{}) {
+				subjobs = append(subjobs, job{
+					key: fmt.Sprintf("%s/%s", in.key, varName),
+					c:   varValue.(cid.Cid),
+				})
+			}
+		}
+	}
+
+	// Walk subfields
+	for _, job := range subjobs {
+		dsc := &dagStatCollector{
+			ds:   dag,
+			walk: carWalkFunc,
+		}
+
+		if err := merkledag.Walk(ctx, dsc.walkLinks, job.c, visit, merkledag.Concurrent()); err != nil {
+			return nil, err
+		}
+		var res result
+		res.key = job.key
+		res.stats = dsc.stats
+
+		results = append(results, res)
+	}
+
+	// now walk the top level object of actor state
+	dsc := &dagStatCollector{
+		ds:   dag,
+		walk: carWalkFunc,
+	}
+
+	if err := merkledag.Walk(ctx, dsc.walkLinks, in.c, visit, merkledag.Concurrent()); err != nil {
+		return nil, err
+	}
+	results = append(results, result{key: in.key, stats: dsc.stats})
+	return results, nil
+}
+
 func collectStats(ctx context.Context, addr address.Address, actor *types.Actor, dag format.NodeGetter) (actorStats, error) {
 	log.Infow("actor", "addr", addr, "code", actor.Code, "name", builtin.ActorNameByCode(actor.Code))
 
@@ -532,3 +940,19 @@ func DumpStats(actStats actorStats) {
 
 	fmt.Println("--------------------------------------------------------------------------")
 }
+
+func DumpSnapshotStats(stats map[string]api.ObjStat) {
+	// sort keys so we get subkey locality
+	keys := make([]string, 0, len(stats))
+	for k := range stats {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	fmt.Printf("%-*s%-*s%-*s\n", 32, "Path", 24, "Size", 24, "\"Blocks\"")
+	for _, k := range keys {
+		stat := stats[k]
+		sizeStr := units.BytesSize(float64(stat.Size))
+		fmt.Printf("%-*s%-*s%-*s%-*d\n", 32, k, 10, sizeStr, 14, fmt.Sprintf("(%d)", stat.Size), 24, stat.Links)
+	}
+}

cmd/lotus-shed/stateroot-stats.go

@@ -197,7 +197,7 @@ var staterootStatCmd = &cli.Command{
 				return err
 			}
 
-			fmt.Printf("%s\t%s\t%d\n", inf.Addr, string(cmh.Digest), inf.Stat.Size)
+			fmt.Printf("%s\t%x\t%d\n", inf.Addr, cmh.Digest, inf.Stat.Size)
 		}
 		return nil
 	},

cmd/lotus-worker/main.go

@@ -8,12 +8,14 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/ipfs/go-datastore/namespace"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/urfave/cli/v2"
 	"go.opencensus.io/stats/view"
@@ -320,6 +322,29 @@ var runCmd = &cli.Command{
 			}
 		}
 
+		// Check DC-environment variable
+		sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"}
+		resourcesType := reflect.TypeOf(storiface.Resources{})
+
+		for _, sectorSize := range sectorSizes {
+			for i := 0; i < resourcesType.NumField(); i++ {
+				field := resourcesType.Field(i)
+				envName := field.Tag.Get("envname")
+				if envName != "" {
+					// Check if DC_[SectorSize]_[ResourceRestriction] is set
+					envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName)
+					if ok {
+						// If it is set, convert it to DC_[ResourceRestriction]
+						err := os.Setenv("DC_"+envName, envVar)
+						if err != nil {
+							log.Fatalf("Error setting environment variable: %v", err)
+						}
+						log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName)
+					}
+				}
+			}
+		}
+
 		// Connect to storage-miner
 		ctx := lcli.ReqContext(cctx)
 
@@ -530,9 +555,14 @@ var runCmd = &cli.Command{
 
 		log.Info("Opening local storage; connecting to master")
 		const unspecifiedAddress = "0.0.0.0"
+
 		address := cctx.String("listen")
-		addressSlice := strings.Split(address, ":")
-		if ip := net.ParseIP(addressSlice[0]); ip != nil {
+		host, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return err
+		}
+
+		if ip := net.ParseIP(host); ip != nil {
 			if ip.String() == unspecifiedAddress {
 				timeout, err := time.ParseDuration(cctx.String("timeout"))
 				if err != nil {
@@ -542,11 +572,21 @@ var runCmd = &cli.Command{
 				if err != nil {
 					return err
 				}
-				address = rip + ":" + addressSlice[1]
+				host = rip
 			}
 		}
 
-		localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + address + "/remote"})
+		var newAddress string
+
+		// Check if the IP address is IPv6
+		ip := net.ParseIP(host)
+		if ip.To4() == nil && ip.To16() != nil {
+			newAddress = "[" + host + "]:" + port
+		} else {
+			newAddress = host + ":" + port
+		}
+
+		localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"})
 		if err != nil {
 			return err
 		}
@@ -587,7 +627,7 @@ var runCmd = &cli.Command{
 			Storage:    lr,
 		}
 
-		log.Info("Setting up control endpoint at " + address)
+		log.Info("Setting up control endpoint at " + newAddress)
 
 		timeout, err := time.ParseDuration(cctx.String("http-server-timeout"))
 		if err != nil {
@@ -612,13 +652,13 @@ var runCmd = &cli.Command{
 			log.Warn("Graceful shutdown successful")
 		}()
 
-		nl, err := net.Listen("tcp", address)
+		nl, err := net.Listen("tcp", newAddress)
 		if err != nil {
 			return err
 		}
 
 		{
-			a, err := net.ResolveTCPAddr("tcp", address)
+			a, err := net.ResolveTCPAddr("tcp", newAddress)
 			if err != nil {
 				return xerrors.Errorf("parsing address: %w", err)
 			}
@@ -699,7 +739,7 @@ var runCmd = &cli.Command{
 
 					select {
 					case <-readyCh:
-						if err := nodeApi.WorkerConnect(ctx, "http://"+address+"/rpc/v0"); err != nil {
+						if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil {
 							log.Errorf("Registering worker failed: %+v", err)
 							cancel()
 							return
@@ -740,21 +780,46 @@ func extractRoutableIP(timeout time.Duration) (string, error) {
 	deprecatedMinerMultiAddrKey := "STORAGE_API_INFO"
 	env, ok := os.LookupEnv(minerMultiAddrKey)
 	if !ok {
-		// TODO remove after deprecation period
 		_, ok = os.LookupEnv(deprecatedMinerMultiAddrKey)
 		if ok {
 			log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey)
 		}
 		return "", xerrors.New("MINER_API_INFO environment variable required to extract IP")
 	}
-	minerAddr := strings.Split(env, "/")
-	conn, err := net.DialTimeout("tcp", minerAddr[2]+":"+minerAddr[4], timeout)
+
+	// Splitting the env to separate the JWT from the multiaddress
+	splitEnv := strings.SplitN(env, ":", 2)
+	if len(splitEnv) < 2 {
+		return "", xerrors.Errorf("invalid MINER_API_INFO format")
+	}
+	// Only take the multiaddress part
+	maddrStr := splitEnv[1]
+
+	maddr, err := multiaddr.NewMultiaddr(maddrStr)
 	if err != nil {
 		return "", err
 	}
-	defer conn.Close() //nolint:errcheck
+
+	minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6)
+	if minerIP == "" {
+		minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4)
+	}
+	minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP)
+
+	// Format the address appropriately
+	addressToDial := net.JoinHostPort(minerIP, minerPort)
+
+	conn, err := net.DialTimeout("tcp", addressToDial, timeout)
+	if err != nil {
+		return "", err
+	}
+
+	defer func() {
+		if cerr := conn.Close(); cerr != nil {
+			log.Errorf("Error closing connection: %v", cerr)
+		}
+	}()
 
 	localAddr := conn.LocalAddr().(*net.TCPAddr)
-
-	return strings.Split(localAddr.IP.String(), ":")[0], nil
+	return localAddr.IP.String(), nil
 }

cmd/lotus/daemon.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"path/filepath"
 	"runtime/pprof"
 	"strings"
 
@@ -32,6 +33,7 @@ import (
 
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/beacon/drand"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/index"
@@ -119,6 +121,10 @@ var DaemonCmd = &cli.Command{
 			Name:  "import-snapshot",
 			Usage: "import chain state from a given chain export file or url",
 		},
+		&cli.BoolFlag{
+			Name:  "remove-existing-chain",
+			Usage: "remove existing chain and splitstore data on a snapshot-import",
+		},
 		&cli.BoolFlag{
 			Name:  "halt-after-import",
 			Usage: "halt the process after importing chain from file",
@@ -263,6 +269,26 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
+		if cctx.Bool("remove-existing-chain") {
+			lr, err := repo.NewFS(cctx.String("repo"))
+			if err != nil {
+				return xerrors.Errorf("error opening fs repo: %w", err)
+			}
+
+			exists, err := lr.Exists()
+			if err != nil {
+				return err
+			}
+			if !exists {
+				return xerrors.Errorf("lotus repo doesn't exist")
+			}
+
+			err = removeExistingChain(cctx, lr)
+			if err != nil {
+				return err
+			}
+		}
+
 		chainfile := cctx.String("import-chain")
 		snapshot := cctx.String("import-snapshot")
 		if chainfile != "" || snapshot != "" {
@@ -380,7 +406,6 @@ var DaemonCmd = &cli.Command{
 		if err != nil {
 			return fmt.Errorf("failed to start json-rpc endpoint: %s", err)
 		}
-
 		// Monitor for shutdown.
 		finishCh := node.MonitorShutdown(shutdownChan,
 			node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper},
@@ -536,13 +561,17 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 		return err
 	}
 
-	// TODO: We need to supply the actual beacon after v14
-	stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
-	if err != nil {
-		return err
-	}
-
 	if !snapshot {
+		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), gb.MinTimestamp(), nil)
+		if err != nil {
+			return xerrors.Errorf("failed to construct beacon schedule: %w", err)
+		}
+
+		stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds, index.DummyMsgIndex)
+		if err != nil {
+			return err
+		}
+
 		log.Infof("validating imported chain...")
 		if err := stm.ValidateChain(ctx, ts); err != nil {
 			return xerrors.Errorf("chain validation failed: %w", err)
@@ -574,3 +603,59 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 
 	return nil
 }
+
+func removeExistingChain(cctx *cli.Context, lr repo.Repo) error {
+	lockedRepo, err := lr.Lock(repo.FullNode)
+	if err != nil {
+		return xerrors.Errorf("error locking repo: %w", err)
+	}
+	// Ensure that lockedRepo is closed when this function exits
+	defer func() {
+		if closeErr := lockedRepo.Close(); closeErr != nil {
+			log.Errorf("Error closing the lockedRepo: %v", closeErr)
+		}
+	}()
+
+	cfg, err := lockedRepo.Config()
+	if err != nil {
+		return xerrors.Errorf("error getting config: %w", err)
+	}
+
+	fullNodeConfig, ok := cfg.(*config.FullNode)
+	if !ok {
+		return xerrors.Errorf("wrong config type: %T", cfg)
+	}
+
+	if fullNodeConfig.Chainstore.EnableSplitstore {
+		log.Info("removing splitstore directory...")
+		err = deleteSplitstoreDir(lockedRepo)
+		if err != nil {
+			return xerrors.Errorf("error removing splitstore directory: %w", err)
+		}
+	}
+
+	// Get the base repo path
+	repoPath := lockedRepo.Path()
+
+	// Construct the path to the chain directory
+	chainPath := filepath.Join(repoPath, "datastore", "chain")
+
+	log.Info("removing chain directory:", chainPath)
+
+	err = os.RemoveAll(chainPath)
+	if err != nil {
+		return xerrors.Errorf("error removing chain directory: %w", err)
+	}
+
+	log.Info("chain and splitstore data have been removed")
+	return nil
+}
+
+func deleteSplitstoreDir(lr repo.LockedRepo) error {
+	path, err := lr.SplitstorePath()
+	if err != nil {
+		return xerrors.Errorf("error getting splitstore path: %w", err)
+	}
+
+	return os.RemoveAll(path)
+}

documentation/en/api-v0-methods.md

@@ -5467,7 +5467,8 @@ Response:
     "UpgradeLightningHeight": 10101,
     "UpgradeThunderHeight": 10101,
     "UpgradeWatermelonHeight": 10101
-  }
+  },
+  "Eip155ChainID": 123
 }
 ```
 

documentation/en/api-v1-unstable-methods.md

@@ -104,6 +104,8 @@
   * [EthSendRawTransaction](#EthSendRawTransaction)
   * [EthSubscribe](#EthSubscribe)
   * [EthSyncing](#EthSyncing)
+  * [EthTraceBlock](#EthTraceBlock)
+  * [EthTraceReplayBlockTransactions](#EthTraceReplayBlockTransactions)
   * [EthUninstallFilter](#EthUninstallFilter)
   * [EthUnsubscribe](#EthUnsubscribe)
 * [Filecoin](#Filecoin)
@@ -2400,14 +2402,7 @@ Perms: read
 Inputs:
 ```json
 [
-  {
-    "from": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
-    "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
-    "gas": "0x5",
-    "gasPrice": "0x0",
-    "value": "0x0",
-    "data": "0x07"
-  }
+  "Bw=="
 ]
 ```
 
@@ -3083,6 +3078,99 @@ Inputs: `null`
 
 Response: `false`
 
+### EthTraceBlock
+TraceAPI related methods
+
+Returns traces created at given block
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  "string value"
+]
+```
+
+Response:
+```json
+[
+  {
+    "action": {
+      "callType": "string value",
+      "from": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+      "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+      "gas": "0x5",
+      "input": "0x07",
+      "value": "0x0"
+    },
+    "result": {
+      "gasUsed": "0x5",
+      "output": "0x07"
+    },
+    "subtraces": 123,
+    "traceAddress": [
+      123
+    ],
+    "Type": "string value",
+    "blockHash": "0x37690cfec6c1bf4c3b9288c7a5d783e98731e90b0a4c177c2a374c7a9427355e",
+    "blockNumber": 9,
+    "transactionHash": "0x37690cfec6c1bf4c3b9288c7a5d783e98731e90b0a4c177c2a374c7a9427355e",
+    "transactionPosition": 123
+  }
+]
+```
+
+### EthTraceReplayBlockTransactions
+Replays all transactions in a block returning the requested traces for each transaction
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  "string value",
+  [
+    "string value"
+  ]
+]
+```
+
+Response:
+```json
+[
+  {
+    "output": "0x07",
+    "stateDiff": "string value",
+    "trace": [
+      {
+        "action": {
+          "callType": "string value",
+          "from": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+          "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+          "gas": "0x5",
+          "input": "0x07",
+          "value": "0x0"
+        },
+        "result": {
+          "gasUsed": "0x5",
+          "output": "0x07"
+        },
+        "subtraces": 123,
+        "traceAddress": [
+          123
+        ],
+        "Type": "string value"
+      }
+    ],
+    "transactionHash": "0x37690cfec6c1bf4c3b9288c7a5d783e98731e90b0a4c177c2a374c7a9427355e",
+    "vmTrace": "string value"
+  }
+]
+```
+
 ### EthUninstallFilter
 Uninstalls a filter with given id.
 
@@ -6981,7 +7069,8 @@ Response:
     "UpgradeLightningHeight": 10101,
     "UpgradeThunderHeight": 10101,
     "UpgradeWatermelonHeight": 10101
-  }
+  },
+  "Eip155ChainID": 123
 }
 ```
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.3
+   1.25.1
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -223,7 +223,6 @@ COMMANDS:
    propose-change-worker       Propose a worker address change
    confirm-change-worker       Confirm a worker address change
    compact-allocated           compact allocated sectors bitfield
-   move-partitions             move deadline of specified partitions from one to another
    propose-change-beneficiary  Propose a beneficiary address change
    confirm-change-beneficiary  Confirm a beneficiary address change
    help, h                     Shows a list of commands or help for one command
@@ -373,22 +372,6 @@ OPTIONS:
    --help, -h                show help
 ```
 
-### lotus-miner actor move-partitions
-```
-NAME:
-   lotus-miner actor move-partitions - move deadline of specified partitions from one to another
-
-USAGE:
-   lotus-miner actor move-partitions [command options] [arguments...]
-
-OPTIONS:
-   --partition-indices value [ --partition-indices value ]  Indices of partitions to update, separated by comma
-   --orig-deadline value                                    Deadline to move partition from (default: 0)
-   --dest-deadline value                                    Deadline to move partition to (default: 0)
-   --really-do-it                                           Actually send transaction performing the action (default: false)
-   --help, -h                                               show help
-```
-
 ### lotus-miner actor propose-change-beneficiary
 ```
 NAME:

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.3
+   1.25.1
 
 COMMANDS:
    run        Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.3
+   1.25.1
 
 COMMANDS:
    daemon   Start a lotus daemon process
@@ -65,6 +65,7 @@ OPTIONS:
    --bootstrap               (default: true)
    --import-chain value      on first run, load chain from given file or url and validate
    --import-snapshot value   import chain state from a given chain export file or url
+   --remove-existing-chain   remove existing chain and splitstore data on a snapshot-import (default: false)
    --halt-after-import       halt the process after importing chain from file (default: false)
    --lite                    start lotus in lite mode (default: false)
    --pprof value             specify name of file for writing cpu profile to

documentation/misc/RELEASE_ISSUE_TEMPLATE.md

@@ -2,28 +2,25 @@
 
 # Lotus X.Y.Z Release
 
-
-## What will be in the release
-
+[//]: # (Open this issue as [WIP] Lotus vX.Y.Z)
+[//]: # (Apply the `tpm` label to it, and pin the issue on GitHub)
 
 ## 🚢 Estimated shipping date
 
 <Date this release will ship on if everything goes to plan (week beginning...)>
 
-## 🔦 Highlights
-
-< See Changelog>
-
 ## ✅ Release Checklist
 
-**Note for whomever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases.  There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end.
+**Note for whoever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases.  There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end.
 
 First steps:
-
+  - [ ] FFI: Fork a new branch (`release/lotus-vX.Y.Z`) from the filecoin-ffi `master` branch
+  - [ ] FFI: Tag the head of `release/lotus-vX.Y.Z` as `vX.Y.Z-pre1`
+  - [ ] Open and land a PR in lotus `master` that updates the FFI dependency to `vX.Y.Z-pre1` as cut in the previous step
   - [ ] Fork a new branch (`release/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage.
   - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release). Run make gen and make docsgen-cli before committing changes
     
-Prepping an RC:
+**Prepping an RC**:
 
 - [ ] version string in `build/version.go` has been updated (in the `release/vX.Y.Z` branch)
 - [ ] run `make gen && make docsgen-cli`
@@ -32,7 +29,7 @@ Prepping an RC:
 - [ ] tag commit with `vX.Y.Z-rcN`
 - [ ] cut a pre-release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=true)
 
-Testing an RC:
+**Testing an RC**:
 
 - [ ] **Stage 0 - Automated Testing**
   - Automated Testing
@@ -69,7 +66,7 @@ Testing an RC:
   - [ ] Update the [CHANGELOG.md](https://github.com/filecoin-project/lotus/blob/master/CHANGELOG.md) to the state that can be used as release note.
   - [ ] Invite the wider community through (link to the release issue)
     
-- [ ] **Stage 4 - Stable Release**
+**Stable Release**
   - [ ] Final preparation
     - [ ] Verify that version string in [`version.go`](https://github.com/filecoin-project/lotus/blob/master/build/version.go) has been updated.
     - [ ] Verify that codegen is up to date (`make gen && make docsgen-cli`)
@@ -79,7 +76,7 @@ Testing an RC:
     - [ ] Cut the release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=false&target=releases).
 
 
-- [ ] **Post-Release**
+**Post-Release**
   - [ ] Merge the `releases` branch back into `master`, ignoring the changes to `version.go` (keep the `-dev` version from master). Do NOT delete the `releases` branch when doing so!
   - [ ] Update [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) with any improvements determined from this latest release iteration.
   - [ ] Create an issue using [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) for the _next_ release.

gateway/node.go

@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
+	verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/filecoin-project/go-state-types/network"
 
@@ -77,6 +78,11 @@ type TargetAPI interface {
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
 	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
+	StateGetAllocationForPendingDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifregtypes.AllocationId, tsk types.TipSetKey) (*verifregtypes.Allocation, error)
+	StateGetAllocations(ctx context.Context, clientAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.AllocationId]verifregtypes.Allocation, error)
+	StateGetClaim(ctx context.Context, providerAddr address.Address, claimId verifregtypes.ClaimId, tsk types.TipSetKey) (*verifregtypes.Claim, error)
+	StateGetClaims(ctx context.Context, providerAddr address.Address, tsk types.TipSetKey) (map[verifregtypes.ClaimId]verifregtypes.Claim, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
@@ -125,7 +131,7 @@ type TargetAPI interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)
@@ -138,6 +144,8 @@ type TargetAPI interface {
 	EthSubscribe(ctx context.Context, params jsonrpc.RawParams) (ethtypes.EthSubscriptionID, error)
 	EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionID) (bool, error)
 	Web3ClientVersion(ctx context.Context) (string, error)
+	EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error)
+	EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error)
 }
 
 var _ TargetAPI = *new(api.FullNode) // gateway depends on latest

gateway/proxy_eth.go

@@ -16,18 +16,11 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/events/filter"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 )
 
-func (gw *Node) Web3ClientVersion(ctx context.Context) (string, error) {
-	if err := gw.limit(ctx, basicRateLimitTokens); err != nil {
-		return "", err
-	}
-
-	return gw.target.Web3ClientVersion(ctx)
-}
-
 func (gw *Node) EthAccounts(ctx context.Context) ([]ethtypes.EthAddress, error) {
 	// gateway provides public API, so it can't hold user accounts
 	return []ethtypes.EthAddress{}, nil
@@ -360,13 +353,19 @@ func (gw *Node) EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt
 	return gw.target.EthMaxPriorityFeePerGas(ctx)
 }
 
-func (gw *Node) EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (gw *Node) EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+	// validate params
+	_, err := jsonrpc.DecodeParams[ethtypes.EthEstimateGasParams](p)
+	if err != nil {
+		return ethtypes.EthUint64(0), xerrors.Errorf("decoding params: %w", err)
+	}
+
 	if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
 		return 0, err
 	}
 
 	// todo limit gas? to what?
-	return gw.target.EthEstimateGas(ctx, tx)
+	return gw.target.EthEstimateGas(ctx, p)
 }
 
 func (gw *Node) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) {
@@ -427,7 +426,7 @@ func (gw *Node) EthGetFilterChanges(ctx context.Context, id ethtypes.EthFilterID
 	ft.lk.Unlock()
 
 	if !ok {
-		return nil, nil
+		return nil, filter.ErrFilterNotFound
 	}
 
 	return gw.target.EthGetFilterChanges(ctx, id)
@@ -581,6 +580,38 @@ func (gw *Node) EthUnsubscribe(ctx context.Context, id ethtypes.EthSubscriptionI
 	return ok, nil
 }
 
+func (gw *Node) Web3ClientVersion(ctx context.Context) (string, error) {
+	if err := gw.limit(ctx, basicRateLimitTokens); err != nil {
+		return "", err
+	}
+
+	return gw.target.Web3ClientVersion(ctx)
+}
+
+func (gw *Node) EthTraceBlock(ctx context.Context, blkNum string) ([]*ethtypes.EthTraceBlock, error) {
+	if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
+		return nil, err
+	}
+
+	if err := gw.checkBlkParam(ctx, blkNum, 0); err != nil {
+		return nil, err
+	}
+
+	return gw.target.EthTraceBlock(ctx, blkNum)
+}
+
+func (gw *Node) EthTraceReplayBlockTransactions(ctx context.Context, blkNum string, traceTypes []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error) {
+	if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
+		return nil, err
+	}
+
+	if err := gw.checkBlkParam(ctx, blkNum, 0); err != nil {
+		return nil, err
+	}
+
+	return gw.target.EthTraceReplayBlockTransactions(ctx, blkNum, traceTypes)
+}
+
 var EthMaxFiltersPerConn = 16 // todo make this configurable
 
 func addUserFilterLimited(ctx context.Context, cb func() (ethtypes.EthFilterID, error)) (ethtypes.EthFilterID, error) {