Compare commits

..
Author SHA1 Message Date
Aayush 065a62af2f Lotus release v1.14.0-rc3 2022-01-21 16:45:51 -05:00
Aayush Rajasekaran 1698257ce9 Merge pull request #7987 from filecoin-project/asr/more-fixes
feat: sealer: some replica update improvements
2022-01-21 16:33:53 -05:00
Aayush 6b3089befc fix: sealer: correctly pipe through errors for SectorAbortUpgrade 2022-01-21 14:07:11 -05:00
Aayush 5e93b642f0 fix: sealer: don't replica update sectors unless they have deals in them 2022-01-21 12:53:36 -05:00
Aayush 0554ca9943 feat: sealer: allow users to abort in-flight snap upgrades 2022-01-21 12:39:18 -05:00
Aayush Rajasekaran 2bc6bdb039 Merge pull request #7985 from filecoin-project/asr/more-fixes
fix: sealer: manager should lock Unsealed for ReplicaUpdate
2022-01-21 12:19:32 -05:00
Aayush 511998195f fix: sealer: manager should lock Unsealed for ReplicaUpdate 2022-01-21 11:10:44 -05:00
Aayush Rajasekaran d8c6c0c563 Merge pull request #7982 from filecoin-project/asr/snap-paths
Fix: sealer: ReplicaUpdate should fetch the correct files
2022-01-20 15:20:57 -05:00
Aayush d066f9613c Fix: sealer: ReplicaUpdate should fetch the correct files 2022-01-20 15:01:26 -05:00
Łukasz Magiera 8e1114a215 Merge pull request #7975 from filecoin-project/asr/update-paths
create replica update paths in acquireSectors
2022-01-19 10:34:17 +01:00
Łukasz Magiera 4cb7552e33 Merge pull request #7976 from filecoin-project/asr/snap-errorcase
fix: checkReplica incorrectly returns ErrBadPR
2022-01-19 10:32:52 +01:00
Aayush Rajasekaran d3123595f2 Merge pull request #7970 from filecoin-project/asr/snap-resource
feat: sealing: Add ReplicaUpdate work to Resource table
2022-01-18 18:58:16 -05:00
Aayush Rajasekaran 895eb476b4 fix: checkReplica incorrectly returns ErrBadPR 2022-01-18 18:54:33 -05:00
Aayush Rajasekaran 5642e2f73f create replica update paths in acquireSectors 2022-01-18 16:35:07 -05:00
Aayush Rajasekaran 60c9b61410 feat: sealing: Add ReplicaUpdate work to Resource table 2022-01-18 10:49:39 -05:00
Aayush Rajasekaran 556e97af17 Merge pull request #7921 from filecoin-project/build/14rcprep
build: release: v1.14.0-rc1
2022-01-12 21:57:50 -05:00
Jennifer Wang a1e0a87a23 add 7939 to change log 2022-01-12 21:52:05 -05:00
Aayush Rajasekaran 3b3a35ee47 Add missing locks to Get() 2022-01-12 21:50:09 -05:00
Jennifer Wang 45c419249c Merge branch 'build/14rcprep' of github.com:filecoin-project/lotus into build/14rcprep 2022-01-12 21:40:51 -05:00
Jennifer Wang 58cf84f072 address reviews 2022-01-12 21:39:29 -05:00
Aayush Rajasekaran 6f9fcc7ef2 Apply suggestions from code review 2022-01-12 21:37:47 -05:00
Jennifer Wang 3505991def fix the statsboard link 2022-01-12 21:33:07 -05:00
Jennifer Wang 9c8a99c8f5 update change log 2022-01-12 21:33:00 -05:00
vyzo 7377cf727e plumb context in tvx 2022-01-12 19:22:12 -05:00
Aayush Rajasekaran 773c20a81c Resolve conflict 2022-01-12 19:16:25 -05:00
Steven Allen ab3c1c507e correctness fixes for the autobatch blockstore
1. Simplify shutdown and make it idempotent by using a context.
2. Make sure `Flush` actually _fully_ flushes if the previous flush failed.
3. Don't clear the flush batch if flushing fails.
2022-01-12 19:09:21 -05:00
Aayush Rajasekaran 4fd49cd541 go mod tidy 2022-01-12 18:59:50 -05:00
Jiaying Wang eba26e6736 update snap net upgrade epoch 2022-01-12 18:59:29 -05:00
Jennifer Wang 065373eb04 update network version for test ground 2022-01-12 18:59:25 -05:00
Jennifer Wang efda3a5847 Update the ntwk v15 name to OhSnap 2022-01-12 18:59:15 -05:00
Jennifer Wang 93c3ec00b1 add butterfly ohsnap epoch 2022-01-12 18:59:06 -05:00
Aayush Rajasekaran f40a7dd7cd Update butterflynet params 2022-01-12 18:58:23 -05:00
Aayush Rajasekaran f2bab4cc63 Update butterflynet params 2022-01-12 18:57:57 -05:00
Jennifer Wang ed72441fbc remove power change 2022-01-12 18:56:53 -05:00
Aayush Rajasekaran 0b6be15fd9 Appease the linter 2022-01-12 18:55:30 -05:00
Jennifer Wang d65b6f6f63 fix lint 2022-01-12 18:55:25 -05:00
Jennifer Wang 0f122df6e2 Check piece before PRU2 instead of PRU1 as PRU2 is the heavy computation part 2022-01-12 18:55:22 -05:00
Jennifer Wang 0f991e54fe just use checkPiece 2022-01-12 18:55:18 -05:00
Jennifer Wang 6854668ff6 Add more deal expiration handling for snap deals 2022-01-12 18:55:12 -05:00
Aayush Rajasekaran 398f0766ba Don't lock in Get 2022-01-12 18:53:57 -05:00
Aayush Rajasekaran 13d9cda20e Address review part 2 2022-01-12 18:53:53 -05:00
Aayush Rajasekaran 98ad7580f8 Address review 2022-01-12 18:53:47 -05:00
Aayush Rajasekaran 4f26f67b3b Support faster Get, retry flushes on error 2022-01-12 18:53:34 -05:00
Aayush Rajasekaran 2ba131e802 Use channels to trigger flushes in a dedicated goroutine 2022-01-12 18:53:29 -05:00
Aayush Rajasekaran b628958cf6 implement stubs 2022-01-12 18:53:21 -05:00
Aayush Rajasekaran a6489f2dd7 cache added cids 2022-01-12 18:53:14 -05:00
Aayush Rajasekaran da39b16c83 Implement an autobatcher 2022-01-12 18:52:17 -05:00
Aayush Rajasekaran def5cb559e Update to actors v7.0.0-rc1 2022-01-12 18:52:12 -05:00
Aayush Rajasekaran bc2171f413 Fast migration for v15 2022-01-12 18:52:07 -05:00
Łukasz Magiera c3cc681e0c fix shed 2022-01-11 20:06:01 -05:00
vyzo e4359eeb19 fix blockstore test contexts 2022-01-11 20:05:01 -05:00
vyzo f9ff542ef4 fix tests 2022-01-11 20:04:38 -05:00
vyzo a536ef227c fix lotus-shed 2022-01-11 20:04:34 -05:00
vyzo 8f31112312 plumb more contexts 2022-01-11 20:04:28 -05:00
Aayush Rajasekaran 908ba3fd5a Deps: Update drand to 1.3.0 2022-01-11 20:03:07 -05:00
Aayush Rajasekaran ace56b5aa8 Deps: Update go-car to v0.3.3 2022-01-11 20:02:55 -05:00
Aayush Rajasekaran 5a2398baa0 Deps: Update lotus-soup to point to tagged releases 2022-01-11 20:02:45 -05:00
Aayush Rajasekaran 6730af0cfe Deps: Update go-fil-markets to v1.13.5 2022-01-11 20:02:29 -05:00
Aayush Rajasekaran 8ef3f6d8b0 Deps: Update go-ipld-cbor to v0.0.6 2022-01-11 20:02:10 -05:00
Aayush Rajasekaran d9361f4be0 Deps: Update go-storedcounter to v0.1.0 2022-01-11 20:01:59 -05:00
Aayush Rajasekaran 1033ec304a Deps: Update dagstore to v0.4.4 2022-01-11 20:01:43 -05:00
Aayush Rajasekaran ec21fd5f2c Deps: Update go-car/v2 to v2.1.1 2022-01-11 20:01:35 -05:00
Aayush Rajasekaran 5a5d3afa80 Deps: Update go-ipld-prime to v0.14.3 2022-01-11 20:01:32 -05:00
Aayush Rajasekaran b5dff4b63c Deps: Update go-ds-badger2 to v0.1.2 2022-01-11 20:01:28 -05:00
Aayush Rajasekaran f8eaa0b8c1 Deps: Update actors v2 to v2.3.6 2022-01-11 19:57:32 -05:00
vyzo e93752b88d fix more paychmgr tests 2022-01-11 19:57:25 -05:00
vyzo beea49df89 fix paychmgr test 2022-01-11 19:57:22 -05:00
Aayush Rajasekaran 11cf83fb88 Deps: Update actors v2 2022-01-11 19:57:17 -05:00
vyzo 9f872d2a94 fix lotus-soup build 2022-01-11 19:57:13 -05:00
vyzo 0e5d5cac82 update lotus-soup deps 2022-01-11 19:56:55 -05:00
vyzo 1ae448f6b7 fix checkpoint test 2022-01-11 19:56:37 -05:00
vyzo 3ec2af0a2b fix messagepool selection test 2022-01-11 19:56:31 -05:00
vyzo ef37ba812e fix messagepool repub test 2022-01-11 19:56:25 -05:00
vyzo 170d2cdfc7 fix messagepool test mock provider 2022-01-11 19:56:19 -05:00
vyzo e98bac367f fix ReorgOps incantation in events test 2022-01-11 19:56:15 -05:00
vyzo 704904e2da fix more tests 2022-01-11 19:56:11 -05:00
vyzo 6317f1422d fix splitstore test 2022-01-11 19:56:07 -05:00
vyzo c8b23b475b fix some unit tests 2022-01-11 19:56:00 -05:00
vyzo 1e07cc6a5a hook a great context in the sky to satisfy DI; sigh... 2022-01-11 19:55:36 -05:00
vyzo 6bca8216bb fix issues with new peerstore constructor signature in DI 2022-01-11 19:55:32 -05:00
vyzo ed9153cb7f fix itests 2022-01-11 19:55:27 -05:00
vyzo 87b21380c7 update drand 2022-01-11 19:55:18 -05:00
vyzo 1eb6f77da1 update for new ConnManager constructor 2022-01-11 19:55:13 -05:00
vyzo 1f99550da8 update go-libp2p-connmgr to v0.3 2022-01-11 19:55:06 -05:00
vyzo 81c8c3f554 update go-libp2p to v0.17 2022-01-11 19:54:59 -05:00
vyzo fa7fa8c199 plumb contexts in lotus-sim 2022-01-11 19:54:19 -05:00
vyzo 16a37036b5 plumb contexts in conformance 2022-01-11 19:54:15 -05:00
vyzo d5b131f4ec go mod tidy 2022-01-11 19:54:08 -05:00
vyzo e0240a4ee5 fix lotus-bench 2022-01-11 19:54:01 -05:00
vyzo 289ea3d33f plumb contexts in lotus-shed 2022-01-11 19:53:54 -05:00
vyzo 074f3850d8 plumb contexts in lotus-miner 2022-01-11 19:53:49 -05:00
vyzo 35b8516288 plumb more contexts in lotus 2022-01-11 19:53:44 -05:00
vyzo d9d4e29253 go get drand@release/v1.3 2022-01-11 19:53:37 -05:00
vyzo 27c9a3b772 update go-ds-measure and go-filestore 2022-01-11 19:52:37 -05:00
Aayush Rajasekaran 489abefce6 Update go-ipld-cbor 2022-01-11 19:51:16 -05:00
Aayush Rajasekaran 5cb33b48da Plumb contexts through 2022-01-11 19:51:08 -05:00
Aayush Rajasekaran df943d3f4e Update markets 2022-01-11 19:50:59 -05:00
Aayush Rajasekaran 7d55284b90 Update carv2 2022-01-11 19:50:34 -05:00
Aayush Rajasekaran 414a53b2df Update go-car 2022-01-11 19:50:27 -05:00
Aayush Rajasekaran 4d98dd9806 Update go-storedcounter and go-ds-versioning 2022-01-11 19:49:48 -05:00
vyzo 585f30a793 update deps 2022-01-11 19:49:23 -05:00
Aayush Rajasekaran dcd7c9ec6a Update go-data-transfer 2022-01-11 19:48:49 -05:00
whyrusleeping a44081aa75 WIP: updating to new datastore/blockstore code with contexts 2022-01-11 19:48:04 -05:00
vyzo c905b6f0c1 satisfy the spellchecker that masquarades as a linter 2022-01-11 19:39:35 -05:00
vyzo e54c8dac56 hook the upgrade schedule to splitstore start 2022-01-11 19:39:31 -05:00
vyzo f7067adf70 add unit test for compaction supression 2022-01-11 19:39:25 -05:00
vyzo ee41547c52 add logic for supressing compaction near upgrade boundaries 2022-01-11 19:39:20 -05:00
Aayush Rajasekaran 07df10a455 format 2022-01-11 18:18:50 -05:00
Jiaying Wang 4bce60bace Update default-lotus-miner-config.toml 2022-01-11 18:17:03 -05:00
Jennifer Wang dbdf16610a fix lint 2022-01-11 18:16:33 -05:00
Jennifer Wang f8a928c4b6 Disable mark-for-upgrade two days before the network v15 OhSnap upgrade to avoid unexpected edge cases that may cause deal/sector failure 2022-01-11 18:16:03 -05:00
Jennifer Wang b5fb1406c5 Integrate proof v11.0.0 2022-01-11 18:07:15 -05:00
Aayush Rajasekaran faa930778f Remove unnecessary params from VerifyWinningPost 2022-01-11 18:06:03 -05:00
Łukasz Magiera 473b333f85 address review 2022-01-11 18:03:41 -05:00
zenground0 b62ccd75a0 Fix hande deal recover return value bug 2022-01-11 18:03:35 -05:00
zenground0 62438885da Fix TooManyMarkedForUpgrade 2022-01-11 18:03:19 -05:00
zenground0 a25eac8033 Deflake more practically 2022-01-11 18:03:14 -05:00
zenground0 e3e4c2ce89 Deflake snap deals integration test 2022-01-11 18:03:10 -05:00
zenground0 8765c69ab8 Snap Deals Integration
- FSM handles the actual cc upgrade process including error states
- PoSting (winning and window) works over upgraded and upgrading sectors
- Integration test and changes to itest framework to reduce flakes
- Update CLI to handle new upgrade
- Update dependencies
2022-01-11 18:02:59 -05:00
Jennifer Wang d665416829 bump the version to v1.14.0-rc1 2022-01-10 23:24:37 -05:00
Aayush Rajasekaran ba85c9fc67 Update go-paramfetch 2022-01-10 23:13:03 -05:00
Jiaying Wang 70c4337835 Update pull_request_template.md 2022-01-10 23:12:12 -05:00
Jiaying Wang 290b31e269 Remove ticket number from the title
issue number in PR title is not that useful because reader cannot click it and need to mention the issue in the body for auto close either way.
2022-01-10 23:11:10 -05:00
Jakub Sztandera 88c23a9cac Add gas charge for VerifyReplicaUpdate
Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
2022-01-10 23:10:20 -05:00
Aayush Rajasekaran 6a06f15650 Fix circsuypply calc around null blocks 2022-01-10 23:10:11 -05:00
Aayush Rajasekaran 762d2e473b VM: Circ supply should be constant per epoch 2022-01-10 23:05:59 -05:00
Aayush Rajasekaran 7c165768d2 Add a log for when message selection fails 2022-01-10 23:04:02 -05:00
Aayush Rajasekaran e19199626a Mempool: add selection tests 2022-01-10 23:03:51 -05:00
Aayush Rajasekaran 3e872599ce Mempool: Selection should respect CBOR limits 2022-01-10 23:03:47 -05:00
Aayush Rajasekaran 518634b998 Consensus: Safety check against unknown sig types 2022-01-10 23:03:41 -05:00
Aayush Rajasekaran d83d585327 Mempool: Selection logic should respect block message limits 2022-01-10 23:03:34 -05:00
Aayush Rajasekaran 7e3fea2489 Update FFI 2022-01-10 23:03:27 -05:00
zenground0 1d3c48c852 RemoveData and Decode
- Unsealing replica update with sector key works and tested
- Sector key generation added and tested
2022-01-10 23:02:44 -05:00
Jiaying Wang 4ae0d1c42e Create pull_request_template.md
This is the very first iteration of the lotus PR template. The goal of adding PR template is to standardize PR requests and encourage contributors to:
- come up with good PR descriptions to give code reviewers a clear overview of what's in the PR 
- have a clear PR title as lotus generates a change log based on it
- check that tests and documentation for the codes that changed are icnluded 

The PR type follows the https://www.conventionalcommits.org/en/v1.0.0-beta.2/.  The [contribution guideline](https://github.com/filecoin-project/lotus#contribute) should be updated with how to create a pr after the template is accepted.
2022-01-10 23:00:08 -05:00
Jennifer Wang 2918a7fa46 docs gen 2022-01-10 22:59:57 -05:00
Jennifer Wang 6bf57a38ec Bump the master version to v1.13.3-dev 2022-01-10 22:55:57 -05:00
zenground0 ef60ee9435 Review Response 2022-01-10 22:50:58 -05:00
zenground0 f7d5efd596 WIP 2022-01-10 22:50:52 -05:00
c r 7c91964cbb reorder transfer checks so as to ensure sending more money than you have to yourself fails with an error (fixing issue 7596)
PR #7637, also adds tests to make sure behavior is correct across versions.
2022-01-10 22:50:08 -05:00
zenground0 a6ce9c13ff WIP sector storage and integration test 2022-01-10 22:49:29 -05:00
Aayush Rajasekaran 0fc5626b3b Address review 2022-01-10 22:44:19 -05:00
Aayush Rajasekaran 9007be1aec Update actors 2022-01-10 22:34:57 -05:00
Aayush Rajasekaran 7884f3ec82 Update FFI 2022-01-10 22:34:53 -05:00
Aayush Rajasekaran 27e21e8db9 Update deps 2022-01-10 22:34:49 -05:00
Aayush Rajasekaran b44596e48f Plug in the FFI call 2022-01-10 22:34:43 -05:00
Aayush Rajasekaran 3e3bd52c51 Integrate v7 actors 2022-01-10 22:34:33 -05:00
271 changed files with 2020 additions and 16208 deletions
+2 -12
View File
@@ -44,13 +44,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
- 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v26-2k-lotus-params'
key: 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
@@ -785,11 +785,6 @@ workflows:
suite: itest-deals_512mb
target: "./itests/deals_512mb_test.go"
- test:
name: test-itest-deals_anycid
suite: itest-deals_anycid
target: "./itests/deals_anycid_test.go"
- test:
name: test-itest-deals_concurrent
suite: itest-deals_concurrent
@@ -855,11 +850,6 @@ workflows:
suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go"
- test:
name: test-itest-mempool
suite: itest-mempool
target: "./itests/mempool_test.go"
- test:
name: test-itest-multisig
suite: itest-multisig
+2 -2
View File
@@ -44,13 +44,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
- 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v26-2k-lotus-params'
key: 'v25-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
+3 -3
View File
@@ -12,10 +12,10 @@
Before you mark the PR ready for review, please make sure that:
- [ ] All commits have a clear commit message.
- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
- [ ] The PR title is in the form of `<PR type>: <area>: <change being made>`
- example: ` fix: mempool: Introduce a cache for valid signatures`
- `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
- `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
- `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_, _misc_,_perf_, _refactor_, _revert_, _style_, _test_
- `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_
- [ ] This PR has tests for new functionality or change in behaviour
- [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
- [ ] CI is green
+34 -51
View File
@@ -1,57 +1,36 @@
# Lotus changelog
# 1.14.2 / 2022-02-24
# 1.14.0-rc3 / 2022-01-12
This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out!
## Bug Fixes
- fix lotus-bench for sealing jobs (#8173)
- fix:sealing:really-do-it flag for abort upgrade (#8181)
- fix:proving:post check sector handles snap deals replica faults (#8177)
- fix: sealing: missing file type (#8180)
## Others
- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong
commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore,
we want to retract it and users may use v1.14.1&^.
## Contributors
| Contributor | Commits | Lines ± | Files Changed |
|-------------|---------|---------|---------------|
| @zenground0 | 2 | +73/-58 | 12 |
| @eben.xie | 1 | +7/-0 | 1 |
| @jennijuju | 1 | +4/-0 | 1 |
| @jennijuju | 1 | +2/-1 | 1 |
| @ribasushi | 1 | +2/-0 | 1 |
# 1.14.1 / 2022-02-18
This is an **optional** release of lotus, that fixes the incorrect *comment* of network v15 OhSnap upgrade **date**. Note the actual upgrade epoch in [v1.14.0](https://github.com/filecoin-project/lotus/releases/tag/v1.14.0) was correct.
# 1.14.0 / 2022-02-17
This is a MANDATORY release of Lotus that introduces [Filecoin network v15,
This is the third release candidate for the mandatory release v1.14.0 of Lotus that introduces [Filecoin network v15,
codenamed the OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
The network is scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, and worker(s).
The OhSnap upgrade introduces the following FIPs, delivered in [actors v7](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0):
The OhSnap upgrade introduces the following FIPs, delivered in [actors v7-rc1](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0-rc1):
- [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
- [FIP-0028 Remove Datacap from Verified clients](https://github.com/filecoin-project/FIPs/pull/226)
It is recommended that storage providers download the new params before updating their node, miner, and workers. To do so:
Note:
- This release is built on top of lotus v1.13.2. Enterprising users like storage providers, data brokers and others
are recommended to test out v1.15.0-rc1(coming on Jan 18th, 2022) for latest new features, improvements and bug
fixes and be ready for the upgrade!
- This release candidate uses temporary proof params for Snap Deals.
- It only sets upgrade epoch for Butterfly-SnapNet, and it does not set the upgrade epochs for calibration and mainnet.
- Download Lotus v1.14.0 or later
- run `make lotus-shed`
- run `./lotus-shed fetch-params` with the appropriate `proving-params` flag
- Upgrade the Lotus daemon and miner **when the previous step is complete**
## Butterfly - SnapNet
All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
The ButterFly-SnapNet will be upgraded to Network v15 OhSnap at epoch 30262, around 2022-01-17T19:00:00Z.
To join the network, simply build lotus by running `make butterflynet`.
The network supports three sector sizes, 512MiB, 32GiB and 64GiB. Temporary proof params for Snap Deals should be downloaded upon your node restarts.
- The parameters are pinged on IPFS gateway https://proofs.filecoin.io/ipfs/ and the CIDs can be found [here](https://github.com/filecoin-project/lotus/blob/edd3486d2cf53b960382e9cda6671e647844aa41/build/proof-params/parameters.json), please let the lotus team know in #lotus-ohsnap if the params are not fetched automatically. You can also download the params manually from s3://proof-params-ap/filecoin-snapdeal-parameters/.
*SnapNet Resources*:
- [Faucet](https://faucet.butterfly.fildev.network/)
- [Stats Dashboard](https://stats.butterfly.fildev.network/d/z6FtI92Zz/chain?orgId=1&refresh=25s&from=now-30m&to=now&kiosk)
- For questions and feedbacks:
- Primary: [Discussion](https://github.com/filecoin-project/lotus/discussions/7935)
- Secondary: #lotus-ohsnap in Filecoin Slack
## New Features and Changes
- Integrate actor v7-rc1:
@@ -86,20 +65,24 @@ All node operators, including storage providers, should be aware that a pre-migr
| Contributor | Commits | Lines ± | Files Changed |
|-------------|---------|---------|---------------|
| Aayush Rajasekaran | 41 | +5538/-1205 | 189 |
| Aayush Rajasekaran | 56 | +5866/-2523 | 316 |
| zenground0 | 11 | +3316/-524 | 124 |
| Jennifer Wang | 29 | +714/-599 | 68 |
| vyzo | 38 | +722/-485 | 117 |
| Rod Vagg | 2 | +6/-941 | 4 |
| whyrusleeping | 2 | +376/-339 | 27 |
| ZenGround0 | 3 | +263/-25 | 11 |
| c r | 2 | +198/-30 | 6 |
| vyzo | 4 | +189/-7 | 7 |
| Aayush | 11 | +146/-48 | 49 |
| Łukasz Magiera | 2 | +184/-3 | 3 |
| Jennifer Wang | 14 | +97/-69 | 41 |
| Whyrusleeping | 1 | +76/-70 | 8 |
| web3-bot | 10 | +99/-17 | 10 |
| Steven Allen | 1 | +55/-37 | 1 |
| Jiaying Wang | 5 | +30/-8 | 5 |
| Hannah Howard | 1 | +4/-29 | 3 |
| dirkmc | 1 | +11/-0 | 1 |
| Jakub Sztandera | 2 | +8/-3 | 3 |
| Łukasz Magiera | 1 | +3/-3 | 2 |
| Travis Person | 1 | +2/-2 | 2 |
| Rod Vagg | 1 | +2/-2 | 2 |
| Colin Kennedy | 1 | +4/-2 | 1 |
# v1.13.2 / 2022-01-09
-2
View File
@@ -345,8 +345,6 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
.PHONY: gen
jen: gen
snap: lotus lotus-miner lotus-worker
snapcraft
# snapcraft upload ./lotus_*.snap
-1
View File
@@ -900,7 +900,6 @@ type QueryOffer struct {
Size uint64
MinPrice types.BigInt
UnsealPrice types.BigInt
PricePerByte abi.TokenAmount
PaymentInterval uint64
PaymentIntervalIncrease uint64
Miner address.Address
+1 -2
View File
@@ -45,9 +45,8 @@ type Gateway interface {
GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-9
View File
@@ -51,15 +51,6 @@ type Net interface {
NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
NetBlockList(ctx context.Context) (NetBlockList, error) //perm:read
NetProtectAdd(ctx context.Context, acl []peer.ID) error //perm:admin
NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
NetProtectList(ctx context.Context) ([]peer.ID, error) //perm:read
// ResourceManager API
NetStat(ctx context.Context, scope string) (NetStat, error) //perm:read
NetLimit(ctx context.Context, scope string) (NetLimit, error) //perm:read
NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error) //perm:read
}
+1 -15
View File
@@ -132,7 +132,6 @@ type StorageMiner interface {
ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true
ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true
@@ -154,7 +153,6 @@ type StorageMiner interface {
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageLocal(ctx context.Context) (map[stores.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
@@ -170,8 +168,6 @@ type StorageMiner interface {
MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error) //perm:read
MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error) //perm:write
MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) //perm:write
// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
@@ -222,16 +218,6 @@ type StorageMiner interface {
// DagstoreGC runs garbage collection on the DAG store.
DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
// so they can download its index
IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
// DagstoreLookupPieces returns information about shards that contain the given CID.
DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
// RuntimeSubsystems returns the subsystems that are enabled
// in this instance.
RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
@@ -266,7 +252,7 @@ type StorageMiner interface {
// the path specified when calling CreateBackup is within the base path
CreateBackup(ctx context.Context, fpath string) error //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
}
-1
View File
@@ -39,7 +39,6 @@ type Worker interface {
SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error) //perm:admin
FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) //perm:admin
ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin
ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin
+1 -40
View File
@@ -1,7 +1,6 @@
package docgen
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
@@ -16,7 +15,6 @@ import (
"github.com/filecoin-project/go-bitfield"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
"github.com/libp2p/go-libp2p-core/metrics"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
@@ -122,7 +120,6 @@ func init() {
addExample(api.FullAPIVersion1)
addExample(api.PCHInbound)
addExample(time.Minute)
addExample(graphsync.RequestID(4))
addExample(datatransfer.TransferID(3))
addExample(datatransfer.Ongoing)
addExample(storeIDExample)
@@ -253,18 +250,10 @@ func init() {
addExample(map[abi.SectorNumber]string{
123: "can't acquire read lock",
})
addExample(json.RawMessage(`"json raw message"`))
addExample(map[api.SectorState]int{
api.SectorState(sealing.Proving): 120,
})
addExample([]abi.SectorNumber{123, 124})
addExample([]storiface.SectorLock{
{
Sector: abi.SectorID{Number: 123, Miner: 1000},
Write: [storiface.FileTypes]uint{0, 0, 1},
Read: [storiface.FileTypes]uint{2, 3, 0},
},
})
// worker specific
addExample(storiface.AcquireMove)
@@ -300,34 +289,6 @@ func init() {
Error: "<error>",
})
addExample(storiface.ResourceTable)
addExample(network.ScopeStat{
Memory: 123,
NumStreamsInbound: 1,
NumStreamsOutbound: 2,
NumConnsInbound: 3,
NumConnsOutbound: 4,
NumFD: 5,
})
addExample(map[string]network.ScopeStat{
"abc": {
Memory: 123,
NumStreamsInbound: 1,
NumStreamsOutbound: 2,
NumConnsInbound: 3,
NumConnsOutbound: 4,
NumFD: 5,
}})
addExample(api.NetLimit{
Memory: 123,
StreamsInbound: 1,
StreamsOutbound: 2,
Streams: 3,
ConnsInbound: 3,
ConnsOutbound: 4,
Conns: 4,
FD: 5,
})
}
func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -378,7 +339,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
switch t.Kind() {
case reflect.Slice:
out := reflect.New(t).Elem()
out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
return out.Interface()
case reflect.Chan:
return ExampleValue(method, t.Elem(), nil)
-87
View File
@@ -1811,21 +1811,6 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
}
// NetLimit mocks base method.
func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
ret0, _ := ret[0].(api.NetLimit)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetLimit indicates an expected call of NetLimit.
func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
}
// NetPeerInfo mocks base method.
func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
m.ctrl.T.Helper()
@@ -1856,49 +1841,6 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
}
// NetProtectAdd mocks base method.
func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectAdd indicates an expected call of NetProtectAdd.
func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
}
// NetProtectList mocks base method.
func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectList", arg0)
ret0, _ := ret[0].([]peer.ID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetProtectList indicates an expected call of NetProtectList.
func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
}
// NetProtectRemove mocks base method.
func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectRemove indicates an expected call of NetProtectRemove.
func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
}
// NetPubsubScores mocks base method.
func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
m.ctrl.T.Helper()
@@ -1914,35 +1856,6 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
}
// NetSetLimit mocks base method.
func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// NetSetLimit indicates an expected call of NetSetLimit.
func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
}
// NetStat mocks base method.
func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
ret0, _ := ret[0].(api.NetStat)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetStat indicates an expected call of NetStat.
func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
}
// NodeStatus mocks base method.
func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) {
m.ctrl.T.Helper()
+4 -186
View File
@@ -516,8 +516,6 @@ type GatewayStruct struct {
MsigGetVested func(p0 context.Context, p1 address.Address, p2 types.TipSetKey, p3 types.TipSetKey) (types.BigInt, error) ``
MsigGetVestingSchedule func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) ``
StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
@@ -589,23 +587,11 @@ type NetStruct struct {
NetFindPeer func(p0 context.Context, p1 peer.ID) (peer.AddrInfo, error) `perm:"read"`
NetLimit func(p0 context.Context, p1 string) (NetLimit, error) `perm:"read"`
NetPeerInfo func(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) `perm:"read"`
NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`
NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
NetProtectRemove func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
NetPubsubScores func(p0 context.Context) ([]PubsubScore, error) `perm:"read"`
NetSetLimit func(p0 context.Context, p1 string, p2 NetLimit) error `perm:"admin"`
NetStat func(p0 context.Context, p1 string) (NetStat, error) `perm:"read"`
}
}
@@ -633,7 +619,7 @@ type StorageMinerStruct struct {
ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"`
CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`
@@ -647,8 +633,6 @@ type StorageMinerStruct struct {
DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`
DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -683,14 +667,8 @@ type StorageMinerStruct struct {
DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
IndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"`
IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
MarketDataTransferDiagnostics func(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) `perm:"write"`
MarketDataTransferUpdates func(p0 context.Context) (<-chan DataTransferChannel, error) `perm:"write"`
MarketGetAsk func(p0 context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
@@ -737,8 +715,6 @@ type StorageMinerStruct struct {
ReturnFetch func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
ReturnFinalizeReplicaUpdate func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
ReturnFinalizeSector func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
ReturnGenerateSectorKeyFromData func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
@@ -831,8 +807,6 @@ type StorageMinerStruct struct {
StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`
StorageGetLocks func(p0 context.Context) (storiface.SectorLocks, error) `perm:"admin"`
StorageInfo func(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) `perm:"admin"`
StorageList func(p0 context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"`
@@ -890,8 +864,6 @@ type WorkerStruct struct {
Fetch func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType, p3 storiface.PathType, p4 storiface.AcquireMode) (storiface.CallID, error) `perm:"admin"`
FinalizeReplicaUpdate func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
FinalizeSector func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
GenerateSectorKeyFromData func(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) `perm:"admin"`
@@ -3299,17 +3271,6 @@ func (s *GatewayStub) MsigGetVested(p0 context.Context, p1 address.Address, p2 t
return *new(types.BigInt), ErrNotSupported
}
func (s *GatewayStruct) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
if s.Internal.MsigGetVestingSchedule == nil {
return *new(MsigVesting), ErrNotSupported
}
return s.Internal.MsigGetVestingSchedule(p0, p1, p2)
}
func (s *GatewayStub) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
return *new(MsigVesting), ErrNotSupported
}
func (s *GatewayStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
if s.Internal.StateAccountKey == nil {
return *new(address.Address), ErrNotSupported
@@ -3662,17 +3623,6 @@ func (s *NetStub) NetFindPeer(p0 context.Context, p1 peer.ID) (peer.AddrInfo, er
return *new(peer.AddrInfo), ErrNotSupported
}
func (s *NetStruct) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
if s.Internal.NetLimit == nil {
return *new(NetLimit), ErrNotSupported
}
return s.Internal.NetLimit(p0, p1)
}
func (s *NetStub) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
return *new(NetLimit), ErrNotSupported
}
func (s *NetStruct) NetPeerInfo(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) {
if s.Internal.NetPeerInfo == nil {
return nil, ErrNotSupported
@@ -3695,39 +3645,6 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
return *new([]peer.AddrInfo), ErrNotSupported
}
func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
if s.Internal.NetProtectAdd == nil {
return ErrNotSupported
}
return s.Internal.NetProtectAdd(p0, p1)
}
func (s *NetStub) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
return ErrNotSupported
}
func (s *NetStruct) NetProtectList(p0 context.Context) ([]peer.ID, error) {
if s.Internal.NetProtectList == nil {
return *new([]peer.ID), ErrNotSupported
}
return s.Internal.NetProtectList(p0)
}
func (s *NetStub) NetProtectList(p0 context.Context) ([]peer.ID, error) {
return *new([]peer.ID), ErrNotSupported
}
func (s *NetStruct) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
if s.Internal.NetProtectRemove == nil {
return ErrNotSupported
}
return s.Internal.NetProtectRemove(p0, p1)
}
func (s *NetStub) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
return ErrNotSupported
}
func (s *NetStruct) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
if s.Internal.NetPubsubScores == nil {
return *new([]PubsubScore), ErrNotSupported
@@ -3739,28 +3656,6 @@ func (s *NetStub) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
return *new([]PubsubScore), ErrNotSupported
}
func (s *NetStruct) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
if s.Internal.NetSetLimit == nil {
return ErrNotSupported
}
return s.Internal.NetSetLimit(p0, p1, p2)
}
func (s *NetStub) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
return ErrNotSupported
}
func (s *NetStruct) NetStat(p0 context.Context, p1 string) (NetStat, error) {
if s.Internal.NetStat == nil {
return *new(NetStat), ErrNotSupported
}
return s.Internal.NetStat(p0, p1)
}
func (s *NetStub) NetStat(p0 context.Context, p1 string) (NetStat, error) {
return *new(NetStat), ErrNotSupported
}
func (s *SignableStruct) Sign(p0 context.Context, p1 SignFunc) error {
if s.Internal.Sign == nil {
return ErrNotSupported
@@ -3805,14 +3700,14 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres
return *new(abi.SectorSize), ErrNotSupported
}
func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
if s.Internal.CheckProvable == nil {
return *new(map[abi.SectorNumber]string), ErrNotSupported
}
return s.Internal.CheckProvable(p0, p1, p2, p3, p4)
return s.Internal.CheckProvable(p0, p1, p2, p3)
}
func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
return *new(map[abi.SectorNumber]string), ErrNotSupported
}
@@ -3882,17 +3777,6 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
return *new([]DagstoreShardInfo), ErrNotSupported
}
func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
if s.Internal.DagstoreLookupPieces == nil {
return *new([]DagstoreShardInfo), ErrNotSupported
}
return s.Internal.DagstoreLookupPieces(p0, p1)
}
func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
return *new([]DagstoreShardInfo), ErrNotSupported
}
func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
if s.Internal.DagstoreRecoverShard == nil {
return ErrNotSupported
@@ -4080,28 +3964,6 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
return ErrNotSupported
}
func (s *StorageMinerStruct) IndexerAnnounceAllDeals(p0 context.Context) error {
if s.Internal.IndexerAnnounceAllDeals == nil {
return ErrNotSupported
}
return s.Internal.IndexerAnnounceAllDeals(p0)
}
func (s *StorageMinerStub) IndexerAnnounceAllDeals(p0 context.Context) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
if s.Internal.IndexerAnnounceDeal == nil {
return ErrNotSupported
}
return s.Internal.IndexerAnnounceDeal(p0, p1)
}
func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
if s.Internal.MarketCancelDataTransfer == nil {
return ErrNotSupported
@@ -4113,17 +3975,6 @@ func (s *StorageMinerStub) MarketCancelDataTransfer(p0 context.Context, p1 datat
return ErrNotSupported
}
func (s *StorageMinerStruct) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
if s.Internal.MarketDataTransferDiagnostics == nil {
return nil, ErrNotSupported
}
return s.Internal.MarketDataTransferDiagnostics(p0, p1)
}
func (s *StorageMinerStub) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
return nil, ErrNotSupported
}
func (s *StorageMinerStruct) MarketDataTransferUpdates(p0 context.Context) (<-chan DataTransferChannel, error) {
if s.Internal.MarketDataTransferUpdates == nil {
return nil, ErrNotSupported
@@ -4377,17 +4228,6 @@ func (s *StorageMinerStub) ReturnFetch(p0 context.Context, p1 storiface.CallID,
return ErrNotSupported
}
func (s *StorageMinerStruct) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
if s.Internal.ReturnFinalizeReplicaUpdate == nil {
return ErrNotSupported
}
return s.Internal.ReturnFinalizeReplicaUpdate(p0, p1, p2)
}
func (s *StorageMinerStub) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
return ErrNotSupported
}
func (s *StorageMinerStruct) ReturnFinalizeSector(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
if s.Internal.ReturnFinalizeSector == nil {
return ErrNotSupported
@@ -4894,17 +4734,6 @@ func (s *StorageMinerStub) StorageFindSector(p0 context.Context, p1 abi.SectorID
return *new([]stores.SectorStorageInfo), ErrNotSupported
}
func (s *StorageMinerStruct) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
if s.Internal.StorageGetLocks == nil {
return *new(storiface.SectorLocks), ErrNotSupported
}
return s.Internal.StorageGetLocks(p0)
}
func (s *StorageMinerStub) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
return *new(storiface.SectorLocks), ErrNotSupported
}
func (s *StorageMinerStruct) StorageInfo(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) {
if s.Internal.StorageInfo == nil {
return *new(stores.StorageInfo), ErrNotSupported
@@ -5125,17 +4954,6 @@ func (s *WorkerStub) Fetch(p0 context.Context, p1 storage.SectorRef, p2 storifac
return *new(storiface.CallID), ErrNotSupported
}
func (s *WorkerStruct) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
if s.Internal.FinalizeReplicaUpdate == nil {
return *new(storiface.CallID), ErrNotSupported
}
return s.Internal.FinalizeReplicaUpdate(p0, p1, p2)
}
func (s *WorkerStub) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
return *new(storiface.CallID), ErrNotSupported
}
func (s *WorkerStruct) FinalizeSector(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
if s.Internal.FinalizeSector == nil {
return *new(storiface.CallID), ErrNotSupported
-48
View File
@@ -10,9 +10,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-graphsync"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
pubsub "github.com/libp2p/go-libp2p-pubsub"
ma "github.com/multiformats/go-multiaddr"
@@ -55,30 +53,6 @@ type MessageSendSpec struct {
MaxFee abi.TokenAmount
}
// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
type GraphSyncDataTransfer struct {
// GraphSync request id for this transfer
RequestID graphsync.RequestID
// Graphsync state for this transfer
RequestState string
// If a channel ID is present, indicates whether this is the current graphsync request for this channel
// (could have changed in a restart)
IsCurrentChannelRequest bool
// Data transfer channel ID for this transfer
ChannelID *datatransfer.ChannelID
// Data transfer state for this transfer
ChannelState *DataTransferChannel
// Diagnostic information about this request -- and unexpected inconsistencies in
// request state
Diagnostics []string
}
// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
type TransferDiagnostics struct {
ReceivingTransfers []*GraphSyncDataTransfer
SendingTransfers []*GraphSyncDataTransfer
}
type DataTransferChannel struct {
TransferID datatransfer.TransferID
Status datatransfer.Status
@@ -130,28 +104,6 @@ type NetBlockList struct {
IPSubnets []string
}
type NetStat struct {
System *network.ScopeStat `json:",omitempty"`
Transient *network.ScopeStat `json:",omitempty"`
Services map[string]network.ScopeStat `json:",omitempty"`
Protocols map[string]network.ScopeStat `json:",omitempty"`
Peers map[string]network.ScopeStat `json:",omitempty"`
}
type NetLimit struct {
Dynamic bool `json:",omitempty"`
// set if Dynamic is false
Memory int64 `json:",omitempty"`
// set if Dynamic is true
MemoryFraction float64 `json:",omitempty"`
MinMemory int64 `json:",omitempty"`
MaxMemory int64 `json:",omitempty"`
Streams, StreamsInbound, StreamsOutbound int
Conns, ConnsInbound, ConnsOutbound int
FD int
}
type ExtendedPeerInfo struct {
ID peer.ID
Agent string
-87
View File
@@ -1724,21 +1724,6 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
}
// NetLimit mocks base method.
func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
ret0, _ := ret[0].(api.NetLimit)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetLimit indicates an expected call of NetLimit.
func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
}
// NetPeerInfo mocks base method.
func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
m.ctrl.T.Helper()
@@ -1769,49 +1754,6 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
}
// NetProtectAdd mocks base method.
func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectAdd indicates an expected call of NetProtectAdd.
func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
}
// NetProtectList mocks base method.
func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectList", arg0)
ret0, _ := ret[0].([]peer.ID)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetProtectList indicates an expected call of NetProtectList.
func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
}
// NetProtectRemove mocks base method.
func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// NetProtectRemove indicates an expected call of NetProtectRemove.
func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
}
// NetPubsubScores mocks base method.
func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
m.ctrl.T.Helper()
@@ -1827,35 +1769,6 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
}
// NetSetLimit mocks base method.
func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// NetSetLimit indicates an expected call of NetSetLimit.
func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
}
// NetStat mocks base method.
func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
ret0, _ := ret[0].(api.NetStat)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NetStat indicates an expected call of NetStat.
func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
}
// PaychAllocateLane mocks base method.
func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
m.ctrl.T.Helper()
+1 -1
View File
@@ -57,7 +57,7 @@ var (
FullAPIVersion0 = newVer(1, 5, 0)
FullAPIVersion1 = newVer(2, 2, 0)
MinerAPIVersion0 = newVer(1, 4, 0)
MinerAPIVersion0 = newVer(1, 3, 0)
WorkerAPIVersion0 = newVer(1, 5, 0)
)
-21
View File
@@ -1,21 +0,0 @@
package blockstore
import (
"context"
)
type hotViewKey struct{}
var hotView = hotViewKey{}
// WithHotView constructs a new context with an option that provides a hint to the blockstore
// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
func WithHotView(ctx context.Context) context.Context {
return context.WithValue(ctx, hotView, struct{}{})
}
// IsHotView returns true if the hot view option is set in the context
func IsHotView(ctx context.Context) bool {
v := ctx.Value(hotView)
return v != nil
}
+4 -11
View File
@@ -49,11 +49,10 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
blockstore and discards writes; this is necessary to support syncing from a snapshot.
- `MarkSetType` -- specifies the type of markset to use during compaction.
The markset is the data structure used by compaction/gc to track live objects.
The default value is "badger", which will use a disk backed markset using badger.
If you have a lot of memory (48G or more) you can also use "map", which will use
an in memory markset, speeding up compaction at the cost of higher memory usage.
Note: If you are using a VPS with a network volume, you need to provision at least
3000 IOPs with the badger markset.
The default value is `"map"`, which will use an in-memory map; if you are limited
in memory (or indeed see compaction run out of memory), you can also specify
`"badger"` which will use an disk backed markset, using badger. This will use
much less memory, but will also make compaction slower.
- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
finalities maintained by default, to maintain messages and message receipts in the
hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -106,12 +105,6 @@ Compaction works transactionally with the following algorithm:
- We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
- We then end the transaction and compact/gc the hotstore.
As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
memory usage; in fact, when using badger as the markset compaction uses very little memory, and
it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
memory during compaction.
## Garbage Collection
TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
-118
View File
@@ -1,118 +0,0 @@
package splitstore
import (
"bufio"
"io"
"os"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)
type Checkpoint struct {
file *os.File
buf *bufio.Writer
}
func NewCheckpoint(path string) (*Checkpoint, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
if err != nil {
return nil, xerrors.Errorf("error creating checkpoint: %w", err)
}
buf := bufio.NewWriter(file)
return &Checkpoint{
file: file,
buf: buf,
}, nil
}
func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
filein, err := os.Open(path)
if err != nil {
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
}
defer filein.Close() //nolint:errcheck
bufin := bufio.NewReader(filein)
start, err := readRawCid(bufin, nil)
if err != nil && err != io.EOF {
return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
}
fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
if err != nil {
return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
}
bufout := bufio.NewWriter(fileout)
return &Checkpoint{
file: fileout,
buf: bufout,
}, start, nil
}
func (cp *Checkpoint) Set(c cid.Cid) error {
if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
}
if err := writeRawCid(cp.buf, c, true); err != nil {
return xerrors.Errorf("error writing cid to checkpoint: %w", err)
}
return nil
}
func (cp *Checkpoint) Close() error {
if cp.file == nil {
return nil
}
err := cp.file.Close()
cp.file = nil
cp.buf = nil
return err
}
func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
sz, err := buf.ReadByte()
if err != nil {
return cid.Undef, err // don't wrap EOF as it is not an error here
}
if hbuf == nil {
hbuf = make([]byte, int(sz))
} else {
hbuf = hbuf[:int(sz)]
}
if _, err := io.ReadFull(buf, hbuf); err != nil {
return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
}
hash, err := mh.Cast(hbuf)
if err != nil {
return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
}
return cid.NewCidV1(cid.Raw, hash), nil
}
func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
hash := c.Hash()
if err := buf.WriteByte(byte(len(hash))); err != nil {
return err
}
if _, err := buf.Write(hash); err != nil {
return err
}
if flush {
return buf.Flush()
}
return nil
}
-147
View File
@@ -1,147 +0,0 @@
package splitstore
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestCheckpoint(t *testing.T) {
dir, err := ioutil.TempDir("", "checkpoint.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(dir)
})
path := filepath.Join(dir, "checkpoint")
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
cp, err := NewCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if err := cp.Set(k1); err != nil {
t.Fatal(err)
}
if err := cp.Set(k2); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err := OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k2) {
t.Fatalf("expected start to be %s; got %s", k2, start)
}
if err := cp.Set(k3); err != nil {
t.Fatal(err)
}
if err := cp.Set(k4); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k4) {
t.Fatalf("expected start to be %s; got %s", k4, start)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
// also test correct operation with an empty checkpoint
cp, err = NewCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if start.Defined() {
t.Fatal("expected start to be undefined")
}
if err := cp.Set(k1); err != nil {
t.Fatal(err)
}
if err := cp.Set(k2); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k2) {
t.Fatalf("expected start to be %s; got %s", k2, start)
}
if err := cp.Set(k3); err != nil {
t.Fatal(err)
}
if err := cp.Set(k4); err != nil {
t.Fatal(err)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
cp, start, err = OpenCheckpoint(path)
if err != nil {
t.Fatal(err)
}
if !start.Equals(k4) {
t.Fatalf("expected start to be %s; got %s", k4, start)
}
if err := cp.Close(); err != nil {
t.Fatal(err)
}
}
-102
View File
@@ -1,102 +0,0 @@
package splitstore
import (
"bufio"
"io"
"os"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type ColdSetWriter struct {
file *os.File
buf *bufio.Writer
}
type ColdSetReader struct {
file *os.File
buf *bufio.Reader
}
func NewColdSetWriter(path string) (*ColdSetWriter, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return nil, xerrors.Errorf("error creating coldset: %w", err)
}
buf := bufio.NewWriter(file)
return &ColdSetWriter{
file: file,
buf: buf,
}, nil
}
func NewColdSetReader(path string) (*ColdSetReader, error) {
file, err := os.Open(path)
if err != nil {
return nil, xerrors.Errorf("error opening coldset: %w", err)
}
buf := bufio.NewReader(file)
return &ColdSetReader{
file: file,
buf: buf,
}, nil
}
func (s *ColdSetWriter) Write(c cid.Cid) error {
return writeRawCid(s.buf, c, false)
}
func (s *ColdSetWriter) Close() error {
if s.file == nil {
return nil
}
err1 := s.buf.Flush()
err2 := s.file.Close()
s.buf = nil
s.file = nil
if err1 != nil {
return err1
}
return err2
}
func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
hbuf := make([]byte, 256)
for {
next, err := readRawCid(s.buf, hbuf)
if err != nil {
if err == io.EOF {
return nil
}
return xerrors.Errorf("error reading coldset: %w", err)
}
if err := f(next); err != nil {
return err
}
}
}
func (s *ColdSetReader) Reset() error {
_, err := s.file.Seek(0, io.SeekStart)
return err
}
func (s *ColdSetReader) Close() error {
if s.file == nil {
return nil
}
err := s.file.Close()
s.file = nil
s.buf = nil
return err
}
-99
View File
@@ -1,99 +0,0 @@
package splitstore
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func TestColdSet(t *testing.T) {
dir, err := ioutil.TempDir("", "coldset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(dir)
})
path := filepath.Join(dir, "coldset")
makeCid := func(i int) cid.Cid {
h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
const count = 1000
cids := make([]cid.Cid, 0, count)
for i := 0; i < count; i++ {
cids = append(cids, makeCid(i))
}
cw, err := NewColdSetWriter(path)
if err != nil {
t.Fatal(err)
}
for _, c := range cids {
if err := cw.Write(c); err != nil {
t.Fatal(err)
}
}
if err := cw.Close(); err != nil {
t.Fatal(err)
}
cr, err := NewColdSetReader(path)
if err != nil {
t.Fatal(err)
}
index := 0
err = cr.ForEach(func(c cid.Cid) error {
if index >= count {
t.Fatal("too many cids")
}
if !c.Equals(cids[index]) {
t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
}
index++
return nil
})
if err != nil {
t.Fatal(err)
}
if err := cr.Reset(); err != nil {
t.Fatal(err)
}
index = 0
err = cr.ForEach(func(c cid.Cid) error {
if index >= count {
t.Fatal("too many cids")
}
if !c.Equals(cids[index]) {
t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
}
index++
return nil
})
if err != nil {
t.Fatal(err)
}
}
+18 -15
View File
@@ -10,36 +10,39 @@ import (
var errMarkSetClosed = errors.New("markset closed")
// MarkSet is an interface for tracking CIDs during chain and object walks
// MarkSet is a utility to keep track of seen CID, and later query for them.
//
// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
// * If a probabilistic result is acceptable, it can be backed by a bloom filter
type MarkSet interface {
ObjectVisitor
Mark(cid.Cid) error
MarkMany([]cid.Cid) error
Has(cid.Cid) (bool, error)
Close() error
SetConcurrent()
}
// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
// of abnormal termination during the critical section span.
BeginCriticalSection() error
// EndCriticalSection ends the critical section span.
EndCriticalSection()
type MarkSetVisitor interface {
MarkSet
ObjectVisitor
}
type MarkSetEnv interface {
// New creates a new markset within the environment.
// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
// Create creates a new markset within the environment.
// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
// sizeHint is a hint about the expected size of the markset
New(name string, sizeHint int64) (MarkSet, error)
// Recover recovers an existing markset persisted on-disk.
Recover(name string) (MarkSet, error)
// Close closes the markset
Create(name string, sizeHint int64) (MarkSet, error)
// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
// It may not be supported by some markset types (e.g. bloom).
CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
SupportsVisitor() bool
Close() error
}
func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
switch mtype {
case "map":
return NewMapMarkSetEnv(path)
return NewMapMarkSetEnv()
case "badger":
return NewBadgerMarkSetEnv(path)
default:
+30 -143
View File
@@ -3,7 +3,6 @@ package splitstore
import (
"os"
"path/filepath"
"runtime"
"sync"
"golang.org/x/xerrors"
@@ -29,13 +28,13 @@ type BadgerMarkSet struct {
writers int
seqno int
version int
persist bool
db *badger.DB
path string
}
var _ MarkSet = (*BadgerMarkSet)(nil)
var _ MarkSetVisitor = (*BadgerMarkSet)(nil)
var badgerMarkSetBatchSize = 16384
@@ -49,10 +48,11 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
return &BadgerMarkSetEnv{path: msPath}, nil
}
func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
name += ".tmp"
path := filepath.Join(e.path, name)
db, err := openBadgerDB(path, false)
db, err := openTransientBadgerDB(path)
if err != nil {
return nil, xerrors.Errorf("error creating badger db: %w", err)
}
@@ -68,72 +68,18 @@ func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
return ms, nil
}
func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
path := filepath.Join(e.path, name)
if _, err := os.Stat(path); err != nil {
return nil, xerrors.Errorf("error stating badger db path: %w", err)
}
db, err := openBadgerDB(path, true)
if err != nil {
return nil, xerrors.Errorf("error creating badger db: %w", err)
}
ms := &BadgerMarkSet{
pend: make(map[string]struct{}),
writing: make(map[int]map[string]struct{}),
db: db,
path: path,
persist: true,
}
ms.cond.L = &ms.mx
return ms, nil
func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
return e.create(name, sizeHint)
}
func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
return e.create(name, sizeHint)
}
func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
func (e *BadgerMarkSetEnv) Close() error {
return nil
}
func (s *BadgerMarkSet) BeginCriticalSection() error {
s.mx.Lock()
if s.persist {
s.mx.Unlock()
return nil
}
var write bool
var seqno int
if len(s.pend) > 0 {
write = true
seqno = s.nextBatch()
}
s.persist = true
s.mx.Unlock()
if write {
// all writes sync once perist is true
return s.write(seqno)
}
// wait for any pending writes and sync
s.mx.Lock()
for s.writers > 0 {
s.cond.Wait()
}
s.mx.Unlock()
return s.db.Sync()
}
func (s *BadgerMarkSet) EndCriticalSection() {
s.mx.Lock()
defer s.mx.Unlock()
s.persist = false
return os.RemoveAll(e.path)
}
func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -153,23 +99,6 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
return nil
}
func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
s.mx.Lock()
if s.pend == nil {
s.mx.Unlock()
return errMarkSetClosed
}
write, seqno := s.putMany(batch)
s.mx.Unlock()
if write {
return s.write(seqno)
}
return nil
}
func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
s.mx.RLock()
defer s.mx.RUnlock()
@@ -275,34 +204,16 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
// writer holds the exclusive lock
func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
s.pend[key] = struct{}{}
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
if len(s.pend) < badgerMarkSetBatchSize {
return false, 0
}
seqno = s.nextBatch()
return true, seqno
}
func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
for _, c := range batch {
key := string(c.Hash())
s.pend[key] = struct{}{}
}
if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
return false, 0
}
seqno = s.nextBatch()
return true, seqno
}
func (s *BadgerMarkSet) nextBatch() int {
seqno := s.seqno
seqno = s.seqno
s.seqno++
s.writing[seqno] = s.pend
s.pend = make(map[string]struct{})
return seqno
return true, seqno
}
func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -347,14 +258,6 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
return xerrors.Errorf("error flushing batch to badger markset: %w", err)
}
s.mx.RLock()
persist := s.persist
s.mx.RUnlock()
if persist {
return s.db.Sync()
}
return nil
}
@@ -374,29 +277,26 @@ func (s *BadgerMarkSet) Close() error {
db := s.db
s.db = nil
return closeBadgerDB(db, s.path, s.persist)
return closeTransientBadgerDB(db, s.path)
}
func openBadgerDB(path string, recover bool) (*badger.DB, error) {
// if it is not a recovery, clean up first
if !recover {
err := os.RemoveAll(path)
if err != nil {
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
}
func (s *BadgerMarkSet) SetConcurrent() {}
err = os.MkdirAll(path, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
func openTransientBadgerDB(path string) (*badger.DB, error) {
// clean up first
err := os.RemoveAll(path)
if err != nil {
return nil, xerrors.Errorf("error clearing markset directory: %w", err)
}
err = os.MkdirAll(path, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
opts := badger.DefaultOptions(path)
// we manually sync when we are in critical section
opts.SyncWrites = false
// no need to do that
opts.CompactL0OnClose = false
// we store hashes, not much to gain by compression
opts.Compression = options.None
// Note: We use FileIO for loading modes to avoid memory thrashing and interference
// between the system blockstore and the markset.
@@ -405,15 +305,6 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
// exceeded 1GB in size.
opts.TableLoadingMode = options.FileIO
opts.ValueLogLoadingMode = options.FileIO
// We increase the number of L0 tables before compaction to make it unlikely to
// be necessary.
opts.NumLevelZeroTables = 20 // default is 5
opts.NumLevelZeroTablesStall = 30 // default is 10
// increase the number of compactors from default 2 so that if we ever have to
// compact, it is fast
if runtime.NumCPU()/2 > opts.NumCompactors {
opts.NumCompactors = runtime.NumCPU() / 2
}
opts.Logger = &badgerLogger{
SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -422,16 +313,12 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
return badger.Open(opts)
}
func closeBadgerDB(db *badger.DB, path string, persist bool) error {
func closeTransientBadgerDB(db *badger.DB, path string) error {
err := db.Close()
if err != nil {
return xerrors.Errorf("error closing badger markset: %w", err)
}
if persist {
return nil
}
err = os.RemoveAll(path)
if err != nil {
return xerrors.Errorf("error deleting badger markset: %w", err)
+35 -219
View File
@@ -1,20 +1,12 @@
package splitstore
import (
"bufio"
"io"
"os"
"path/filepath"
"sync"
"golang.org/x/xerrors"
cid "github.com/ipfs/go-cid"
)
type MapMarkSetEnv struct {
path string
}
type MapMarkSetEnv struct{}
var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
@@ -22,194 +14,55 @@ type MapMarkSet struct {
mx sync.RWMutex
set map[string]struct{}
persist bool
file *os.File
buf *bufio.Writer
path string
ts bool
}
var _ MarkSet = (*MapMarkSet)(nil)
var _ MarkSetVisitor = (*MapMarkSet)(nil)
func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
msPath := filepath.Join(path, "markset.map")
err := os.MkdirAll(msPath, 0755) //nolint:gosec
if err != nil {
return nil, xerrors.Errorf("error creating markset directory: %w", err)
}
return &MapMarkSetEnv{path: msPath}, nil
func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
return &MapMarkSetEnv{}, nil
}
func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
path := filepath.Join(e.path, name)
func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
return &MapMarkSet{
set: make(map[string]struct{}, sizeHint),
path: path,
set: make(map[string]struct{}, sizeHint),
}, nil
}
func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
path := filepath.Join(e.path, name)
s := &MapMarkSet{
set: make(map[string]struct{}),
path: path,
}
in, err := os.Open(path)
if err != nil {
return nil, xerrors.Errorf("error opening markset file for read: %w", err)
}
defer in.Close() //nolint:errcheck
// wrap a buffered reader to make this faster
buf := bufio.NewReader(in)
for {
var sz byte
if sz, err = buf.ReadByte(); err != nil {
break
}
key := make([]byte, int(sz))
if _, err = io.ReadFull(buf, key); err != nil {
break
}
s.set[string(key)] = struct{}{}
}
if err != io.EOF {
return nil, xerrors.Errorf("error reading markset file: %w", err)
}
file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
return nil, xerrors.Errorf("error opening markset file for write: %w", err)
}
s.persist = true
s.file = file
s.buf = bufio.NewWriter(file)
return s, nil
func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
return e.create(name, sizeHint)
}
func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
return e.create(name, sizeHint)
}
func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
func (e *MapMarkSetEnv) Close() error {
return nil
}
func (s *MapMarkSet) BeginCriticalSection() error {
s.mx.Lock()
defer s.mx.Unlock()
func (s *MapMarkSet) Mark(cid cid.Cid) error {
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.set == nil {
return errMarkSetClosed
}
if s.persist {
return nil
}
file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return xerrors.Errorf("error opening markset file: %w", err)
}
// wrap a buffered writer to make this faster
s.buf = bufio.NewWriter(file)
for key := range s.set {
if err := s.writeKey([]byte(key), false); err != nil {
_ = file.Close()
s.buf = nil
return err
}
}
if err := s.buf.Flush(); err != nil {
_ = file.Close()
s.buf = nil
return xerrors.Errorf("error flushing markset file buffer: %w", err)
}
s.file = file
s.persist = true
return nil
}
func (s *MapMarkSet) EndCriticalSection() {
s.mx.Lock()
defer s.mx.Unlock()
if !s.persist {
return
}
_ = s.file.Close()
_ = os.Remove(s.path)
s.file = nil
s.buf = nil
s.persist = false
}
func (s *MapMarkSet) Mark(c cid.Cid) error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return errMarkSetClosed
}
hash := c.Hash()
s.set[string(hash)] = struct{}{}
if s.persist {
if err := s.writeKey(hash, true); err != nil {
return err
}
if err := s.file.Sync(); err != nil {
return xerrors.Errorf("error syncing markset: %w", err)
}
}
return nil
}
func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return errMarkSetClosed
}
for _, c := range batch {
hash := c.Hash()
s.set[string(hash)] = struct{}{}
if s.persist {
if err := s.writeKey(hash, false); err != nil {
return err
}
}
}
if s.persist {
if err := s.buf.Flush(); err != nil {
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
}
if err := s.file.Sync(); err != nil {
return xerrors.Errorf("error syncing markset: %w", err)
}
}
s.set[string(cid.Hash())] = struct{}{}
return nil
}
func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
s.mx.RLock()
defer s.mx.RUnlock()
if s.ts {
s.mx.RLock()
defer s.mx.RUnlock()
}
if s.set == nil {
return false, errMarkSetClosed
@@ -220,70 +73,33 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
}
func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
s.mx.Lock()
defer s.mx.Unlock()
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
if s.set == nil {
return false, errMarkSetClosed
}
hash := c.Hash()
key := string(hash)
key := string(c.Hash())
if _, ok := s.set[key]; ok {
return false, nil
}
s.set[key] = struct{}{}
if s.persist {
if err := s.writeKey(hash, true); err != nil {
return false, err
}
if err := s.file.Sync(); err != nil {
return false, xerrors.Errorf("error syncing markset: %w", err)
}
}
return true, nil
}
func (s *MapMarkSet) Close() error {
s.mx.Lock()
defer s.mx.Unlock()
if s.set == nil {
return nil
if s.ts {
s.mx.Lock()
defer s.mx.Unlock()
}
s.set = nil
if s.file != nil {
if err := s.file.Close(); err != nil {
log.Warnf("error closing markset file: %s", err)
}
if !s.persist {
if err := os.Remove(s.path); err != nil {
log.Warnf("error removing markset file: %s", err)
}
}
}
return nil
}
func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
if err := s.buf.WriteByte(byte(len(k))); err != nil {
return xerrors.Errorf("error writing markset key length to disk: %w", err)
}
if _, err := s.buf.Write(k); err != nil {
return xerrors.Errorf("error writing markset key to disk: %w", err)
}
if flush {
if err := s.buf.Flush(); err != nil {
return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
}
}
return nil
func (s *MapMarkSet) SetConcurrent() {
s.ts = true
}
+9 -332
View File
@@ -11,10 +11,7 @@ import (
func TestMapMarkSet(t *testing.T) {
testMarkSet(t, "map")
testMarkSetRecovery(t, "map")
testMarkSetMarkMany(t, "map")
testMarkSetVisitor(t, "map")
testMarkSetVisitorRecovery(t, "map")
}
func TestBadgerMarkSet(t *testing.T) {
@@ -24,13 +21,12 @@ func TestBadgerMarkSet(t *testing.T) {
badgerMarkSetBatchSize = bs
})
testMarkSet(t, "badger")
testMarkSetRecovery(t, "badger")
testMarkSetMarkMany(t, "badger")
testMarkSetVisitor(t, "badger")
testMarkSetVisitorRecovery(t, "badger")
}
func testMarkSet(t *testing.T, lsType string) {
t.Helper()
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
@@ -46,12 +42,12 @@ func testMarkSet(t *testing.T, lsType string) {
}
defer env.Close() //nolint:errcheck
hotSet, err := env.New("hot", 0)
hotSet, err := env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err := env.New("cold", 0)
coldSet, err := env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
@@ -66,7 +62,6 @@ func testMarkSet(t *testing.T, lsType string) {
}
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
@@ -78,7 +73,6 @@ func testMarkSet(t *testing.T, lsType string) {
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
@@ -120,12 +114,12 @@ func testMarkSet(t *testing.T, lsType string) {
t.Fatal(err)
}
hotSet, err = env.New("hot", 0)
hotSet, err = env.Create("hot", 0)
if err != nil {
t.Fatal(err)
}
coldSet, err = env.New("cold", 0)
coldSet, err = env.Create("cold", 0)
if err != nil {
t.Fatal(err)
}
@@ -156,6 +150,8 @@ func testMarkSet(t *testing.T, lsType string) {
}
func testMarkSetVisitor(t *testing.T, lsType string) {
t.Helper()
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
@@ -171,7 +167,7 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
}
defer env.Close() //nolint:errcheck
visitor, err := env.New("test", 0)
visitor, err := env.CreateVisitor("test", 0)
if err != nil {
t.Fatal(err)
}
@@ -223,322 +219,3 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
}
func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
visitor, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
defer visitor.Close() //nolint:errcheck
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustVisit := func(v ObjectVisitor, cid cid.Cid) {
visit, err := v.Visit(cid)
if err != nil {
t.Fatal(err)
}
if !visit {
t.Fatal("object should be visited")
}
}
mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
visit, err := v.Visit(cid)
if err != nil {
t.Fatal(err)
}
if visit {
t.Fatal("unexpected visit")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
mustVisit(visitor, k1)
mustVisit(visitor, k2)
if err := visitor.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
mustVisit(visitor, k3)
mustVisit(visitor, k4)
mustNotVisit(visitor, k1)
mustNotVisit(visitor, k2)
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
if err := visitor.Close(); err != nil {
t.Fatal(err)
}
visitor, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustNotVisit(visitor, k1)
mustNotVisit(visitor, k2)
mustNotVisit(visitor, k3)
mustNotVisit(visitor, k4)
visitor.EndCriticalSection()
if err := visitor.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
func testMarkSetRecovery(t *testing.T, lsType string) {
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
markSet, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("mark not found")
}
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unexpected mark")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
if err := markSet.Mark(k1); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k2); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustNotHave(markSet, k3)
mustNotHave(markSet, k4)
if err := markSet.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k3); err != nil {
t.Fatal(err)
}
if err := markSet.Mark(k4); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
markSet, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
markSet.EndCriticalSection()
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
func testMarkSetMarkMany(t *testing.T, lsType string) {
path, err := ioutil.TempDir("", "markset.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
env, err := OpenMarkSetEnv(path, lsType)
if err != nil {
t.Fatal(err)
}
defer env.Close() //nolint:errcheck
markSet, err := env.New("test", 0)
if err != nil {
t.Fatal(err)
}
makeCid := func(key string) cid.Cid {
h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
if err != nil {
t.Fatal(err)
}
return cid.NewCidV1(cid.Raw, h)
}
mustHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatal("mark not found")
}
}
mustNotHave := func(s MarkSet, cid cid.Cid) {
t.Helper()
has, err := s.Has(cid)
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("unexpected mark")
}
}
k1 := makeCid("a")
k2 := makeCid("b")
k3 := makeCid("c")
k4 := makeCid("d")
if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustNotHave(markSet, k3)
mustNotHave(markSet, k4)
if err := markSet.BeginCriticalSection(); err != nil {
t.Fatal(err)
}
if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
markSet, err = env.Recover("test")
if err != nil {
t.Fatal(err)
}
mustHave(markSet, k1)
mustHave(markSet, k2)
mustHave(markSet, k3)
mustHave(markSet, k4)
markSet.EndCriticalSection()
if err := markSet.Close(); err != nil {
t.Fatal(err)
}
_, err = env.Recover("test")
if err == nil {
t.Fatal("expected recovery to fail")
}
}
+10 -124
View File
@@ -129,6 +129,8 @@ type SplitStore struct {
headChangeMx sync.Mutex
coldPurgeSize int
chain ChainAccessor
ds dstore.Datastore
cold bstore.Blockstore
@@ -156,17 +158,6 @@ type SplitStore struct {
txnRefsMx sync.Mutex
txnRefs map[cid.Cid]struct{}
txnMissing map[cid.Cid]struct{}
txnMarkSet MarkSet
txnSyncMx sync.Mutex
txnSyncCond sync.Cond
txnSync bool
// background cold object reification
reifyWorkers sync.WaitGroup
reifyMx sync.Mutex
reifyCond sync.Cond
reifyPend map[cid.Cid]struct{}
reifyInProgress map[cid.Cid]struct{}
// registered protectors
protectors []func(func(cid.Cid) error) error
@@ -195,6 +186,10 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
return nil, err
}
if !markSetEnv.SupportsVisitor() {
return nil, xerrors.Errorf("markset type does not support atomic visitors")
}
// and now we can make a SplitStore
ss := &SplitStore{
cfg: cfg,
@@ -203,16 +198,13 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
cold: cold,
hot: hots,
markSetEnv: markSetEnv,
coldPurgeSize: defaultColdPurgeSize,
}
ss.txnViewsCond.L = &ss.txnViewsMx
ss.txnSyncCond.L = &ss.txnSyncMx
ss.ctx, ss.cancel = context.WithCancel(context.Background())
ss.reifyCond.L = &ss.reifyMx
ss.reifyPend = make(map[cid.Cid]struct{})
ss.reifyInProgress = make(map[cid.Cid]struct{})
if enableDebugLog {
ss.debug, err = openDebugLog(path)
if err != nil {
@@ -220,14 +212,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
}
}
if ss.checkpointExists() {
log.Info("found compaction checkpoint; resuming compaction")
if err := ss.completeCompaction(); err != nil {
markSetEnv.Close() //nolint:errcheck
return nil, xerrors.Errorf("error resuming compaction: %w", err)
}
}
return ss, nil
}
@@ -250,20 +234,6 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return false, err
}
if has {
return s.has(cid)
}
return s.cold.Has(ctx, cid)
}
has, err := s.hot.Has(ctx, cid)
if err != nil {
@@ -275,13 +245,7 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
return true, nil
}
has, err = s.cold.Has(ctx, cid)
if has && bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
return has, err
return s.cold.Has(ctx, cid)
}
func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
@@ -297,20 +261,6 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return nil, err
}
if has {
return s.get(cid)
}
return s.cold.Get(ctx, cid)
}
blk, err := s.hot.Get(ctx, cid)
switch err {
@@ -325,11 +275,8 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
blk, err = s.cold.Get(ctx, cid)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return blk, err
@@ -351,20 +298,6 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
// critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
if err != nil {
return 0, err
}
if has {
return s.getSize(cid)
}
return s.cold.GetSize(ctx, cid)
}
size, err := s.hot.GetSize(ctx, cid)
switch err {
@@ -379,10 +312,6 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
size, err = s.cold.GetSize(ctx, cid)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return size, err
@@ -407,12 +336,6 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
s.debug.LogWrite(blk)
// critical section
if s.txnMarkSet != nil {
s.markLiveRefs([]cid.Cid{blk.Cid()})
return nil
}
s.trackTxnRef(blk.Cid())
return nil
}
@@ -458,12 +381,6 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
s.debug.LogWriteMany(blks)
// critical section
if s.txnMarkSet != nil {
s.markLiveRefs(batch)
return nil
}
s.trackTxnRefMany(batch)
return nil
}
@@ -523,23 +440,6 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
return cb(data)
}
// critical section
s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
if s.txnMarkSet != nil {
has, err := s.txnMarkSet.Has(cid)
s.txnLk.RUnlock()
if err != nil {
return err
}
if has {
return s.view(cid, cb)
}
return s.cold.View(ctx, cid, cb)
}
// views are (optimistically) protected two-fold:
// - if there is an active transaction, then the reference is protected.
// - if there is no active transaction, active views are tracked in a
@@ -560,10 +460,6 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
err = s.cold.View(ctx, cid, cb)
if err == nil {
if bstore.IsHotView(ctx) {
s.reifyColdObject(cid)
}
stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
}
return err
@@ -673,9 +569,6 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
}
}
// spawn the reifier
go s.reifyOrchestrator()
// watch the chain
chain.SubscribeHeadChanges(s.HeadChange)
@@ -696,19 +589,12 @@ func (s *SplitStore) Close() error {
}
if atomic.LoadInt32(&s.compacting) == 1 {
s.txnSyncMx.Lock()
s.txnSync = true
s.txnSyncCond.Broadcast()
s.txnSyncMx.Unlock()
log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
for atomic.LoadInt32(&s.compacting) == 1 {
time.Sleep(time.Second)
}
}
s.reifyCond.Broadcast()
s.reifyWorkers.Wait()
s.cancel()
return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
}
+6 -11
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
@@ -68,10 +67,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
defer output.Close() //nolint:errcheck
var mx sync.Mutex
write := func(format string, args ...interface{}) {
mx.Lock()
defer mx.Unlock()
_, err := fmt.Fprintf(output, format+"\n", args...)
if err != nil {
log.Warnf("error writing check output: %s", err)
@@ -86,10 +82,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
write("compaction index: %d", s.compactionIndex)
write("--")
coldCnt := new(int64)
missingCnt := new(int64)
var coldCnt, missingCnt int64
visitor, err := s.markSetEnv.New("check", 0)
visitor, err := s.markSetEnv.CreateVisitor("check", 0)
if err != nil {
return xerrors.Errorf("error creating visitor: %w", err)
}
@@ -116,10 +111,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
}
if has {
atomic.AddInt64(coldCnt, 1)
coldCnt++
write("cold object reference: %s", c)
} else {
atomic.AddInt64(missingCnt, 1)
missingCnt++
write("missing object reference: %s", c)
return errStopWalk
}
@@ -133,9 +128,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
return err
}
log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
write("--")
write("cold: %d missing: %d", *coldCnt, *missingCnt)
write("cold: %d missing: %d", coldCnt, missingCnt)
write("DONE")
return nil
+204 -488
View File
@@ -3,10 +3,8 @@ package splitstore
import (
"bytes"
"errors"
"os"
"path/filepath"
"runtime"
"sync"
"sort"
"sync/atomic"
"time"
@@ -49,10 +47,6 @@ var (
// SyncGapTime is the time delay from a tipset's min timestamp before we decide
// there is a sync gap
SyncGapTime = time.Minute
// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
// we have synced.
SyncWaitTime = 30 * time.Second
)
var (
@@ -62,6 +56,8 @@ var (
const (
batchSize = 16384
defaultColdPurgeSize = 7_000_000
)
func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -144,9 +140,9 @@ func (s *SplitStore) isNearUpgrade(epoch abi.ChainEpoch) bool {
// transactionally protect incoming tipsets
func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
s.txnLk.RLock()
defer s.txnLk.RUnlock()
if !s.txnActive {
s.txnLk.RUnlock()
return
}
@@ -155,115 +151,12 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
cids = append(cids, ts.Cids()...)
}
if len(cids) == 0 {
s.txnLk.RUnlock()
return
}
// critical section
if s.txnMarkSet != nil {
curTs := apply[len(apply)-1]
timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
doSync := time.Since(timestamp) < SyncWaitTime
go func() {
if doSync {
defer func() {
s.txnSyncMx.Lock()
defer s.txnSyncMx.Unlock()
s.txnSync = true
s.txnSyncCond.Broadcast()
}()
}
defer s.txnLk.RUnlock()
s.markLiveRefs(cids)
}()
return
}
s.trackTxnRefMany(cids)
s.txnLk.RUnlock()
}
func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
log.Debugf("marking %d live refs", len(cids))
startMark := time.Now()
count := new(int32)
visitor := newConcurrentVisitor()
walkObject := func(c cid.Cid) error {
return s.walkObjectIncomplete(c, visitor,
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
visit, err := s.txnMarkSet.Visit(c)
if err != nil {
return xerrors.Errorf("error visiting object: %w", err)
}
if !visit {
return errStopWalk
}
atomic.AddInt32(count, 1)
return nil
},
func(missing cid.Cid) error {
log.Warnf("missing object reference %s in %s", missing, c)
return errStopWalk
})
}
// optimize the common case of single put
if len(cids) == 1 {
if err := walkObject(cids[0]); err != nil {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
return
}
workch := make(chan cid.Cid, len(cids))
for _, c := range cids {
workch <- c
}
close(workch)
worker := func() error {
for c := range workch {
if err := walkObject(c); err != nil {
return err
}
}
return nil
}
workers := runtime.NumCPU() / 2
if workers < 2 {
workers = 2
}
if workers > len(cids) {
workers = len(cids)
}
g := new(errgroup.Group)
for i := 0; i < workers; i++ {
g.Go(worker)
}
if err := g.Wait(); err != nil {
log.Errorf("error marking tipset refs: %s", err)
}
log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
}
// transactionally protect a view
func (s *SplitStore) protectView(c cid.Cid) {
// the txnLk is held for read
s.txnLk.RLock()
defer s.txnLk.RUnlock()
if s.txnActive {
@@ -334,7 +227,7 @@ func (s *SplitStore) trackTxnRefMany(cids []cid.Cid) {
}
// protect all pending transactional references
func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
func (s *SplitStore) protectTxnRefs(markSet MarkSetVisitor) error {
for {
var txnRefs map[cid.Cid]struct{}
@@ -406,14 +299,14 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
// transactionally protect a reference by walking the object and marking.
// concurrent markings are short circuited by checking the markset.
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSetVisitor) error {
if err := s.checkClosing(); err != nil {
return err
}
// Note: cold objects are deleted heaviest first, so the consituents of an object
// cannot be deleted before the object itself.
return s.walkObjectIncomplete(root, newTmpVisitor(),
return s.walkObjectIncomplete(root, tmpVisitor(),
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
@@ -493,12 +386,6 @@ func (s *SplitStore) compact(curTs *types.TipSet) {
}
func (s *SplitStore) doCompact(curTs *types.TipSet) error {
if s.checkpointExists() {
// this really shouldn't happen, but if it somehow does, it means that the hotstore
// might be potentially inconsistent; abort compaction and notify the user to intervene.
return xerrors.Errorf("checkpoint exists; aborting compaction")
}
currentEpoch := curTs.Height()
boundaryEpoch := currentEpoch - CompactionBoundary
@@ -510,7 +397,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "inclMsgsEpoch", inclMsgsEpoch, "compactionIndex", s.compactionIndex)
markSet, err := s.markSetEnv.New("live", s.markSetSize)
markSet, err := s.markSetEnv.CreateVisitor("live", s.markSetSize)
if err != nil {
return xerrors.Errorf("error creating mark set: %w", err)
}
@@ -521,6 +408,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return err
}
// we are ready for concurrent marking
s.beginTxnMarking(markSet)
// 0. track all protected references at beginning of compaction; anything added later should
// be transactionally protected by the write
log.Info("protecting references with registered protectors")
@@ -534,7 +424,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
log.Info("marking reachable objects")
startMark := time.Now()
count := new(int64)
var count int64
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
func(c cid.Cid) error {
if isUnitaryObject(c) {
@@ -550,7 +440,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return errStopWalk
}
atomic.AddInt64(count, 1)
count++
return nil
})
@@ -558,9 +448,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return xerrors.Errorf("error marking: %w", err)
}
s.markSetSize = *count + *count>>2 // overestimate a bit
s.markSetSize = count + count>>2 // overestimate a bit
log.Infow("marking done", "took", time.Since(startMark), "marked", *count)
log.Infow("marking done", "took", time.Since(startMark), "marked", count)
if err := s.checkClosing(); err != nil {
return err
@@ -580,15 +470,10 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
log.Info("collecting cold objects")
startCollect := time.Now()
coldw, err := NewColdSetWriter(s.coldSetPath())
if err != nil {
return xerrors.Errorf("error creating coldset: %w", err)
}
defer coldw.Close() //nolint:errcheck
// some stats for logging
var hotCnt, coldCnt int
cold := make([]cid.Cid, 0, s.coldPurgeSize)
err = s.hot.ForEachKey(func(c cid.Cid) error {
// was it marked?
mark, err := markSet.Has(c)
@@ -602,9 +487,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
}
// it's cold, mark it as candidate for move
if err := coldw.Write(c); err != nil {
return xerrors.Errorf("error writing cid to coldstore: %w", err)
}
cold = append(cold, c)
coldCnt++
return nil
@@ -614,12 +497,12 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return xerrors.Errorf("error collecting cold objects: %w", err)
}
if err := coldw.Close(); err != nil {
return xerrors.Errorf("error closing coldset: %w", err)
}
log.Infow("cold collection done", "took", time.Since(startCollect))
if coldCnt > 0 {
s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
}
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
@@ -637,17 +520,11 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
return err
}
coldr, err := NewColdSetReader(s.coldSetPath())
if err != nil {
return xerrors.Errorf("error opening coldset: %w", err)
}
defer coldr.Close() //nolint:errcheck
// 3. copy the cold objects to the coldstore -- if we have one
if !s.cfg.DiscardColdBlocks {
log.Info("moving cold objects to the coldstore")
startMove := time.Now()
err = s.moveColdBlocks(coldr)
err = s.moveColdBlocks(cold)
if err != nil {
return xerrors.Errorf("error moving cold objects: %w", err)
}
@@ -656,64 +533,41 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
if err := s.checkClosing(); err != nil {
return err
}
if err := coldr.Reset(); err != nil {
return xerrors.Errorf("error resetting coldset: %w", err)
}
}
// 4. Purge cold objects with checkpointing for recovery.
// This is the critical section of compaction, whereby any cold object not in the markSet is
// considered already deleted.
// We delete cold objects in batches, holding the transaction lock, where we check the markSet
// again for new references created by the VM.
// After each batch, we write a checkpoint to disk; if the process is interrupted before completion,
// the process will continue from the checkpoint in the next recovery.
if err := s.beginCriticalSection(markSet); err != nil {
return xerrors.Errorf("error beginning critical section: %w", err)
}
if err := s.checkClosing(); err != nil {
return err
}
// wait for the head to catch up so that the current tipset is marked
s.waitForSync()
if err := s.checkClosing(); err != nil {
return err
}
checkpoint, err := NewCheckpoint(s.checkpointPath())
// 4. sort cold objects so that the dags with most references are deleted first
// this ensures that we can't refer to a dag with its consituents already deleted, ie
// we lave no dangling references.
log.Info("sorting cold objects")
startSort := time.Now()
err = s.sortObjects(cold)
if err != nil {
return xerrors.Errorf("error creating checkpoint: %w", err)
return xerrors.Errorf("error sorting objects: %w", err)
}
log.Infow("sorting done", "took", time.Since(startSort))
// 4.1 protect transactional refs once more
// strictly speaking, this is not necessary as purge will do it before deleting each
// batch. however, there is likely a largish number of references accumulated during
// ths sort and this protects before entering pruge context.
err = s.protectTxnRefs(markSet)
if err != nil {
return xerrors.Errorf("error protecting transactional refs: %w", err)
}
if err := s.checkClosing(); err != nil {
return err
}
defer checkpoint.Close() //nolint:errcheck
// 5. purge cold objects from the hotstore, taking protected references into account
log.Info("purging cold objects from the hotstore")
startPurge := time.Now()
err = s.purge(coldr, checkpoint, markSet)
err = s.purge(cold, markSet)
if err != nil {
return xerrors.Errorf("error purging cold objects: %w", err)
return xerrors.Errorf("error purging cold blocks: %w", err)
}
log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
s.endCriticalSection()
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
}
if err := os.Remove(s.checkpointPath()); err != nil {
log.Warnf("error removing checkpoint: %s", err)
}
if err := coldr.Close(); err != nil {
log.Warnf("error closing coldset: %s", err)
}
if err := os.Remove(s.coldSetPath()); err != nil {
log.Warnf("error removing coldset: %s", err)
}
// we are done; do some housekeeping
s.endTxnProtect()
s.gcHotstore()
@@ -744,51 +598,12 @@ func (s *SplitStore) beginTxnProtect() {
defer s.txnLk.Unlock()
s.txnActive = true
s.txnSync = false
s.txnRefs = make(map[cid.Cid]struct{})
s.txnMissing = make(map[cid.Cid]struct{})
}
func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
log.Info("beginning critical section")
// do that once first to get the bulk before the markset is in critical section
if err := s.protectTxnRefs(markSet); err != nil {
return xerrors.Errorf("error protecting transactional references: %w", err)
}
if err := markSet.BeginCriticalSection(); err != nil {
return xerrors.Errorf("error beginning critical section for markset: %w", err)
}
s.txnLk.Lock()
defer s.txnLk.Unlock()
s.txnMarkSet = markSet
// and do it again while holding the lock to mark references that might have been created
// in the meantime and avoid races of the type Has->txnRef->enterCS->Get fails because
// it's not in the markset
if err := s.protectTxnRefs(markSet); err != nil {
return xerrors.Errorf("error protecting transactional references: %w", err)
}
return nil
}
func (s *SplitStore) waitForSync() {
log.Info("waiting for sync")
startWait := time.Now()
defer func() {
log.Infow("waiting for sync done", "took", time.Since(startWait))
}()
s.txnSyncMx.Lock()
defer s.txnSyncMx.Unlock()
for !s.txnSync {
s.txnSyncCond.Wait()
}
func (s *SplitStore) beginTxnMarking(markSet MarkSetVisitor) {
markSet.SetConcurrent()
}
func (s *SplitStore) endTxnProtect() {
@@ -800,51 +615,32 @@ func (s *SplitStore) endTxnProtect() {
}
s.txnActive = false
s.txnSync = false
s.txnRefs = nil
s.txnMissing = nil
s.txnMarkSet = nil
}
func (s *SplitStore) endCriticalSection() {
log.Info("ending critical section")
s.txnLk.Lock()
defer s.txnLk.Unlock()
s.txnMarkSet.EndCriticalSection()
s.txnMarkSet = nil
}
func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
visitor ObjectVisitor, f func(cid.Cid) error) error {
var walked ObjectVisitor
var mx sync.Mutex
// we copy the tipset first into a new slice, which allows us to reuse it in every epoch.
toWalk := make([]cid.Cid, len(ts.Cids()))
copy(toWalk, ts.Cids())
walkCnt := new(int64)
scanCnt := new(int64)
var walked *cid.Set
toWalk := ts.Cids()
walkCnt := 0
scanCnt := 0
stopWalk := func(_ cid.Cid) error { return errStopWalk }
walkBlock := func(c cid.Cid) error {
visit, err := walked.Visit(c)
if err != nil {
return err
}
if !visit {
if !walked.Visit(c) {
return nil
}
atomic.AddInt64(walkCnt, 1)
walkCnt++
if err := f(c); err != nil {
return err
}
var hdr types.BlockHeader
err = s.view(c, func(data []byte) error {
err := s.view(c, func(data []byte) error {
return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
})
@@ -880,13 +676,11 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
if err := s.walkObject(hdr.ParentStateRoot, visitor, f); err != nil {
return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
}
atomic.AddInt64(scanCnt, 1)
scanCnt++
}
if hdr.Height > 0 {
mx.Lock()
toWalk = append(toWalk, hdr.Parents...)
mx.Unlock()
}
return nil
@@ -898,43 +692,20 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
return err
}
workers := len(toWalk)
if workers > runtime.NumCPU()/2 {
workers = runtime.NumCPU() / 2
}
if workers < 2 {
workers = 2
}
// the walk is BFS, so we can reset the walked set in every iteration and avoid building up
// a set that contains all blocks (1M epochs -> 5M blocks -> 200MB worth of memory and growing
// over time)
walked = newConcurrentVisitor()
workch := make(chan cid.Cid, len(toWalk))
for _, c := range toWalk {
workch <- c
}
close(workch)
toWalk = toWalk[:0]
g := new(errgroup.Group)
for i := 0; i < workers; i++ {
g.Go(func() error {
for c := range workch {
if err := walkBlock(c); err != nil {
return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
}
}
return nil
})
}
if err := g.Wait(); err != nil {
return err
walked = cid.NewSet()
walking := toWalk
toWalk = nil
for _, c := range walking {
if err := walkBlock(c); err != nil {
return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
}
}
}
log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt)
log.Infow("chain walk done", "walked", walkCnt, "scanned", scanCnt)
return nil
}
@@ -1053,7 +824,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
return nil
}
// internal version used during compaction and related operations
// internal version used by walk
func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
if isIdentiyCid(c) {
data, err := decodeIdentityCid(c)
@@ -1088,34 +859,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
return s.cold.Has(s.ctx, c)
}
func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
blk, err := s.hot.Get(s.ctx, c)
switch err {
case nil:
return blk, nil
case bstore.ErrNotFound:
return s.cold.Get(s.ctx, c)
default:
return nil, err
}
}
func (s *SplitStore) getSize(c cid.Cid) (int, error) {
sz, err := s.hot.GetSize(s.ctx, c)
switch err {
case nil:
return sz, nil
case bstore.ErrNotFound:
return s.cold.GetSize(s.ctx, c)
default:
return 0, err
}
}
func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
batch := make([]blocks.Block, 0, batchSize)
err := coldr.ForEach(func(c cid.Cid) error {
for _, c := range cold {
if err := s.checkClosing(); err != nil {
return err
}
@@ -1124,7 +871,7 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
if err != nil {
if err == bstore.ErrNotFound {
log.Warnf("hotstore missing block %s", c)
return nil
continue
}
return xerrors.Errorf("error retrieving block %s from hotstore: %w", c, err)
@@ -1138,12 +885,6 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
}
batch = batch[:0]
}
return nil
})
if err != nil {
return xerrors.Errorf("error iterating coldset: %w", err)
}
if len(batch) > 0 {
@@ -1156,202 +897,177 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
return nil
}
func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet MarkSet) error {
batch := make([]cid.Cid, 0, batchSize)
deadCids := make([]cid.Cid, 0, batchSize)
var purgeCnt, liveCnt int
defer func() {
log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
}()
deleteBatch := func() error {
pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
purgeCnt += pc
liveCnt += lc
batch = batch[:0]
return err
// sorts a slice of objects heaviest first -- it's a little expensive but worth the
// guarantee that we don't leave dangling references behind, e.g. if we die in the middle
// of a purge.
func (s *SplitStore) sortObjects(cids []cid.Cid) error {
// we cache the keys to avoid making a gazillion of strings
keys := make(map[cid.Cid]string)
key := func(c cid.Cid) string {
s, ok := keys[c]
if !ok {
s = string(c.Hash())
keys[c] = s
}
return s
}
err := coldr.ForEach(func(c cid.Cid) error {
batch = append(batch, c)
if len(batch) == batchSize {
return deleteBatch()
// compute sorting weights as the cumulative number of DAG links
weights := make(map[string]int)
for _, c := range cids {
// this can take quite a while, so check for shutdown with every opportunity
if err := s.checkClosing(); err != nil {
return err
}
return nil
w := s.getObjectWeight(c, weights, key)
weights[key(c)] = w
}
// sort!
sort.Slice(cids, func(i, j int) bool {
wi := weights[key(cids[i])]
wj := weights[key(cids[j])]
if wi == wj {
return bytes.Compare(cids[i].Hash(), cids[j].Hash()) > 0
}
return wi > wj
})
if err != nil {
return err
}
if len(batch) > 0 {
return deleteBatch()
}
return nil
}
func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoint, markSet MarkSet) (purgeCnt int, liveCnt int, err error) {
if err := s.checkClosing(); err != nil {
return 0, 0, err
func (s *SplitStore) getObjectWeight(c cid.Cid, weights map[string]int, key func(cid.Cid) string) int {
w, ok := weights[key(c)]
if ok {
return w
}
s.txnLk.Lock()
defer s.txnLk.Unlock()
// we treat block headers specially to avoid walking the entire chain
var hdr types.BlockHeader
err := s.view(c, func(data []byte) error {
return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
})
if err == nil {
w1 := s.getObjectWeight(hdr.ParentStateRoot, weights, key)
weights[key(hdr.ParentStateRoot)] = w1
for _, c := range batch {
has, err := markSet.Has(c)
if err != nil {
return 0, 0, xerrors.Errorf("error checking markset for liveness: %w", err)
}
w2 := s.getObjectWeight(hdr.Messages, weights, key)
weights[key(hdr.Messages)] = w2
if has {
liveCnt++
return 1 + w1 + w2
}
var links []cid.Cid
err = s.view(c, func(data []byte) error {
return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
links = append(links, c)
})
})
if err != nil {
return 1
}
w = 1
for _, c := range links {
// these are internal refs, so dags will be dags
if c.Prefix().Codec != cid.DagCBOR {
w++
continue
}
deadCids = append(deadCids, c)
wc := s.getObjectWeight(c, weights, key)
weights[key(c)] = wc
w += wc
}
if len(deadCids) == 0 {
if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
return 0, 0, xerrors.Errorf("error setting checkpoint: %w", err)
return w
}
func (s *SplitStore) purgeBatch(cids []cid.Cid, deleteBatch func([]cid.Cid) error) error {
if len(cids) == 0 {
return nil
}
// we don't delete one giant batch of millions of objects, but rather do smaller batches
// so that we don't stop the world for an extended period of time
done := false
for i := 0; !done; i++ {
start := i * batchSize
end := start + batchSize
if end >= len(cids) {
end = len(cids)
done = true
}
return 0, liveCnt, nil
err := deleteBatch(cids[start:end])
if err != nil {
return xerrors.Errorf("error deleting batch: %w", err)
}
}
if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
}
s.debug.LogDelete(deadCids)
purgeCnt = len(deadCids)
if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
return purgeCnt, liveCnt, xerrors.Errorf("error setting checkpoint: %w", err)
}
return purgeCnt, liveCnt, nil
}
func (s *SplitStore) coldSetPath() string {
return filepath.Join(s.path, "coldset")
}
func (s *SplitStore) checkpointPath() string {
return filepath.Join(s.path, "checkpoint")
}
func (s *SplitStore) checkpointExists() bool {
_, err := os.Stat(s.checkpointPath())
return err == nil
}
func (s *SplitStore) completeCompaction() error {
checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
if err != nil {
return xerrors.Errorf("error opening checkpoint: %w", err)
}
defer checkpoint.Close() //nolint:errcheck
coldr, err := NewColdSetReader(s.coldSetPath())
if err != nil {
return xerrors.Errorf("error opening coldset: %w", err)
}
defer coldr.Close() //nolint:errcheck
markSet, err := s.markSetEnv.Recover("live")
if err != nil {
return xerrors.Errorf("error recovering markset: %w", err)
}
defer markSet.Close() //nolint:errcheck
// PURGE
log.Info("purging cold objects from the hotstore")
startPurge := time.Now()
err = s.completePurge(coldr, checkpoint, last, markSet)
if err != nil {
return xerrors.Errorf("error purging cold objects: %w", err)
}
log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
markSet.EndCriticalSection()
if err := checkpoint.Close(); err != nil {
log.Warnf("error closing checkpoint: %s", err)
}
if err := os.Remove(s.checkpointPath()); err != nil {
log.Warnf("error removing checkpoint: %s", err)
}
if err := coldr.Close(); err != nil {
log.Warnf("error closing coldset: %s", err)
}
if err := os.Remove(s.coldSetPath()); err != nil {
log.Warnf("error removing coldset: %s", err)
}
// Note: at this point we can start the splitstore; a compaction should run on
// the first head change, which will trigger gc on the hotstore.
// We don't mind the second (back-to-back) compaction as the head will
// have advanced during marking and coldset accumulation.
return nil
}
func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, start cid.Cid, markSet MarkSet) error {
if !start.Defined() {
return s.purge(coldr, checkpoint, markSet)
}
seeking := true
batch := make([]cid.Cid, 0, batchSize)
func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSetVisitor) error {
deadCids := make([]cid.Cid, 0, batchSize)
var purgeCnt, liveCnt int
defer func() {
log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
}()
deleteBatch := func() error {
pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
return s.purgeBatch(cids,
func(cids []cid.Cid) error {
deadCids := deadCids[:0]
purgeCnt += pc
liveCnt += lc
batch = batch[:0]
for {
if err := s.checkClosing(); err != nil {
return err
}
return err
}
s.txnLk.Lock()
if len(s.txnRefs) == 0 {
// keep the lock!
break
}
err := coldr.ForEach(func(c cid.Cid) error {
if seeking {
if start.Equals(c) {
seeking = false
// unlock and protect
s.txnLk.Unlock()
err := s.protectTxnRefs(markSet)
if err != nil {
return xerrors.Errorf("error protecting transactional refs: %w", err)
}
}
defer s.txnLk.Unlock()
for _, c := range cids {
live, err := markSet.Has(c)
if err != nil {
return xerrors.Errorf("error checking for liveness: %w", err)
}
if live {
liveCnt++
continue
}
deadCids = append(deadCids, c)
}
err := s.hot.DeleteMany(s.ctx, deadCids)
if err != nil {
return xerrors.Errorf("error purging cold objects: %w", err)
}
s.debug.LogDelete(deadCids)
purgeCnt += len(deadCids)
return nil
}
batch = append(batch, c)
if len(batch) == batchSize {
return deleteBatch()
}
return nil
})
if err != nil {
return err
}
if len(batch) > 0 {
return deleteBatch()
}
return nil
})
}
// I really don't like having this code, but we seem to have some occasional DAG references with
@@ -1361,7 +1077,7 @@ func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint,
// have this gem[TM].
// My best guess is that they are parent message receipts or yet to be computed state roots; magik
// thinks the cause may be block validation.
func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
func (s *SplitStore) waitForMissingRefs(markSet MarkSetVisitor) {
s.txnLk.Lock()
missing := s.txnMissing
s.txnMissing = nil
@@ -1390,7 +1106,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
}
towalk := missing
visitor := newTmpVisitor()
visitor := tmpVisitor()
missing = make(map[cid.Cid]struct{})
for c := range towalk {
-214
View File
@@ -1,214 +0,0 @@
package splitstore
import (
"errors"
"runtime"
"sync/atomic"
"golang.org/x/xerrors"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
)
var (
errReifyLimit = errors.New("reification limit reached")
ReifyLimit = 16384
)
func (s *SplitStore) reifyColdObject(c cid.Cid) {
if !s.isWarm() {
return
}
if isUnitaryObject(c) {
return
}
s.reifyMx.Lock()
defer s.reifyMx.Unlock()
_, ok := s.reifyInProgress[c]
if ok {
return
}
s.reifyPend[c] = struct{}{}
s.reifyCond.Broadcast()
}
func (s *SplitStore) reifyOrchestrator() {
workers := runtime.NumCPU() / 4
if workers < 2 {
workers = 2
}
workch := make(chan cid.Cid, workers)
defer close(workch)
for i := 0; i < workers; i++ {
s.reifyWorkers.Add(1)
go s.reifyWorker(workch)
}
for {
s.reifyMx.Lock()
for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
s.reifyCond.Wait()
}
if atomic.LoadInt32(&s.closing) != 0 {
s.reifyMx.Unlock()
return
}
reifyPend := s.reifyPend
s.reifyPend = make(map[cid.Cid]struct{})
s.reifyMx.Unlock()
for c := range reifyPend {
select {
case workch <- c:
case <-s.ctx.Done():
return
}
}
}
}
func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
defer s.reifyWorkers.Done()
for c := range workch {
s.doReify(c)
}
}
func (s *SplitStore) doReify(c cid.Cid) {
var toreify, totrack, toforget []cid.Cid
defer func() {
s.reifyMx.Lock()
defer s.reifyMx.Unlock()
for _, c := range toreify {
delete(s.reifyInProgress, c)
}
for _, c := range totrack {
delete(s.reifyInProgress, c)
}
for _, c := range toforget {
delete(s.reifyInProgress, c)
}
}()
s.txnLk.RLock()
defer s.txnLk.RUnlock()
count := 0
err := s.walkObjectIncomplete(c, newTmpVisitor(),
func(c cid.Cid) error {
if isUnitaryObject(c) {
return errStopWalk
}
count++
if count > ReifyLimit {
return errReifyLimit
}
s.reifyMx.Lock()
_, inProgress := s.reifyInProgress[c]
if !inProgress {
s.reifyInProgress[c] = struct{}{}
}
s.reifyMx.Unlock()
if inProgress {
return errStopWalk
}
has, err := s.hot.Has(s.ctx, c)
if err != nil {
return xerrors.Errorf("error checking hotstore: %w", err)
}
if has {
if s.txnMarkSet != nil {
hasMark, err := s.txnMarkSet.Has(c)
if err != nil {
log.Warnf("error checking markset: %s", err)
} else if hasMark {
toforget = append(toforget, c)
return errStopWalk
}
} else {
totrack = append(totrack, c)
return errStopWalk
}
}
toreify = append(toreify, c)
return nil
},
func(missing cid.Cid) error {
log.Warnf("missing reference while reifying %s: %s", c, missing)
return errStopWalk
})
if err != nil {
if xerrors.Is(err, errReifyLimit) {
log.Debug("reification aborted; reify limit reached")
return
}
log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
return
}
log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
// this should not get too big, maybe some 100s of objects.
batch := make([]blocks.Block, 0, len(toreify))
for _, c := range toreify {
blk, err := s.cold.Get(s.ctx, c)
if err != nil {
log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
continue
}
if err := s.checkClosing(); err != nil {
return
}
batch = append(batch, blk)
}
if len(batch) > 0 {
err = s.hot.PutMany(s.ctx, batch)
if err != nil {
log.Warnf("error reifying cold object (cid: %s): %s", c, err)
return
}
}
if s.txnMarkSet != nil {
if len(toreify) > 0 {
if err := s.txnMarkSet.MarkMany(toreify); err != nil {
log.Warnf("error marking reified objects: %s", err)
}
}
if len(totrack) > 0 {
if err := s.txnMarkSet.MarkMany(totrack); err != nil {
log.Warnf("error marking tracked objects: %s", err)
}
}
} else {
// if txnActive is false these are noops
if len(toreify) > 0 {
s.trackTxnRefMany(toreify)
}
if len(totrack) > 0 {
s.trackTxnRefMany(totrack)
}
}
}
+11 -279
View File
@@ -4,9 +4,6 @@ import (
"context"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"sync/atomic"
"testing"
@@ -23,14 +20,12 @@ import (
datastore "github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
logging "github.com/ipfs/go-log/v2"
mh "github.com/multiformats/go-multihash"
)
func init() {
CompactionThreshold = 5
CompactionBoundary = 2
WarmupBoundary = 0
SyncWaitTime = time.Millisecond
logging.SetLogLevel("splitstore", "DEBUG")
}
@@ -85,17 +80,8 @@ func testSplitStore(t *testing.T, cfg *Config) {
t.Fatal(err)
}
path, err := ioutil.TempDir("", "splitstore.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
// open the splitstore
ss, err := Open(path, ds, hot, cold, cfg)
ss, err := Open("", ds, hot, cold, cfg)
if err != nil {
t.Fatal(err)
}
@@ -139,10 +125,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
}
waitForCompaction := func() {
ss.txnSyncMx.Lock()
ss.txnSync = true
ss.txnSyncCond.Broadcast()
ss.txnSyncMx.Unlock()
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
}
@@ -277,17 +259,8 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
t.Fatal(err)
}
path, err := ioutil.TempDir("", "splitstore.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
// open the splitstore
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
ss, err := Open("", ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
@@ -332,10 +305,6 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
}
waitForCompaction := func() {
ss.txnSyncMx.Lock()
ss.txnSync = true
ss.txnSyncCond.Broadcast()
ss.txnSyncMx.Unlock()
for atomic.LoadInt32(&ss.compacting) == 1 {
time.Sleep(100 * time.Millisecond)
}
@@ -388,235 +357,6 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
}
}
func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := newMockStore()
cold := newMockStore()
mkRandomBlock := func() blocks.Block {
data := make([]byte, 128)
_, err := rand.Read(data)
if err != nil {
t.Fatal(err)
}
return blocks.NewBlock(data)
}
block1 := mkRandomBlock()
block2 := mkRandomBlock()
block3 := mkRandomBlock()
hdr := mock.MkBlock(nil, 0, 0)
hdr.Messages = block1.Cid()
hdr.ParentMessageReceipts = block2.Cid()
hdr.ParentStateRoot = block3.Cid()
block4, err := hdr.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
allBlocks := []blocks.Block{block1, block2, block3, block4}
for _, blk := range allBlocks {
err := cold.Put(context.Background(), blk)
if err != nil {
t.Fatal(err)
}
}
path, err := ioutil.TempDir("", "splitstore.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
ss.warmupEpoch = 1
go ss.reifyOrchestrator()
waitForReification := func() {
for {
ss.reifyMx.Lock()
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
ss.reifyMx.Unlock()
if ready {
return
}
time.Sleep(time.Millisecond)
}
}
// first access using the standard view
err = f(context.Background(), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
// nothing should be reified
waitForReification()
for _, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("block unexpectedly reified")
}
}
// now make the hot/reifying view and ensure access reifies
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
// everything should be reified
waitForReification()
for i, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if !has {
t.Fatalf("block%d was not reified", i+1)
}
}
}
func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
ds := dssync.MutexWrap(datastore.NewMapDatastore())
hot := newMockStore()
cold := newMockStore()
mkRandomBlock := func() blocks.Block {
data := make([]byte, 128)
_, err := rand.Read(data)
if err != nil {
t.Fatal(err)
}
return blocks.NewBlock(data)
}
block1 := mkRandomBlock()
block2 := mkRandomBlock()
block3 := mkRandomBlock()
hdr := mock.MkBlock(nil, 0, 0)
hdr.Messages = block1.Cid()
hdr.ParentMessageReceipts = block2.Cid()
hdr.ParentStateRoot = block3.Cid()
block4, err := hdr.ToStorageBlock()
if err != nil {
t.Fatal(err)
}
allBlocks := []blocks.Block{block1, block2, block3, block4}
for _, blk := range allBlocks {
err := cold.Put(context.Background(), blk)
if err != nil {
t.Fatal(err)
}
}
path, err := ioutil.TempDir("", "splitstore.*")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = os.RemoveAll(path)
})
ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
if err != nil {
t.Fatal(err)
}
defer ss.Close() //nolint
ss.warmupEpoch = 1
go ss.reifyOrchestrator()
waitForReification := func() {
for {
ss.reifyMx.Lock()
ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
ss.reifyMx.Unlock()
if ready {
return
}
time.Sleep(time.Millisecond)
}
}
// do a hot access -- nothing should be reified as the limit should be exceeded
oldReifyLimit := ReifyLimit
ReifyLimit = 2
t.Cleanup(func() {
ReifyLimit = oldReifyLimit
})
err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
if err != nil {
t.Fatal(err)
}
waitForReification()
for _, blk := range allBlocks {
has, err := hot.Has(context.Background(), blk.Cid())
if err != nil {
t.Fatal(err)
}
if has {
t.Fatal("block unexpectedly reified")
}
}
}
func TestSplitStoreReification(t *testing.T) {
t.Log("test reification with Has")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Has(ctx, c)
return err
})
t.Log("test reification with Get")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Get(ctx, c)
return err
})
t.Log("test reification with GetSize")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.GetSize(ctx, c)
return err
})
t.Log("test reification with View")
testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
return s.View(ctx, c, func(_ []byte) error { return nil })
})
t.Log("test reification limit")
testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
_, err := s.Has(ctx, c)
return err
})
}
type mockChain struct {
t testing.TB
@@ -686,25 +426,17 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
type mockStore struct {
mx sync.Mutex
set map[string]blocks.Block
set map[cid.Cid]blocks.Block
}
func newMockStore() *mockStore {
return &mockStore{set: make(map[string]blocks.Block)}
}
func (b *mockStore) keyOf(c cid.Cid) string {
return string(c.Hash())
}
func (b *mockStore) cidOf(k string) cid.Cid {
return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
return &mockStore{set: make(map[cid.Cid]blocks.Block)}
}
func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
b.mx.Lock()
defer b.mx.Unlock()
_, ok := b.set[b.keyOf(cid)]
_, ok := b.set[cid]
return ok, nil
}
@@ -714,7 +446,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
b.mx.Lock()
defer b.mx.Unlock()
blk, ok := b.set[b.keyOf(cid)]
blk, ok := b.set[cid]
if !ok {
return nil, blockstore.ErrNotFound
}
@@ -742,7 +474,7 @@ func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
b.mx.Lock()
defer b.mx.Unlock()
b.set[b.keyOf(blk.Cid())] = blk
b.set[blk.Cid()] = blk
return nil
}
@@ -751,7 +483,7 @@ func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
defer b.mx.Unlock()
for _, blk := range blks {
b.set[b.keyOf(blk.Cid())] = blk
b.set[blk.Cid()] = blk
}
return nil
}
@@ -760,7 +492,7 @@ func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
b.mx.Lock()
defer b.mx.Unlock()
delete(b.set, b.keyOf(cid))
delete(b.set, cid)
return nil
}
@@ -769,7 +501,7 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
defer b.mx.Unlock()
for _, c := range cids {
delete(b.set, b.keyOf(c))
delete(b.set, c)
}
return nil
}
@@ -783,7 +515,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
defer b.mx.Unlock()
for c := range b.set {
err := f(b.cidOf(c))
err := f(c)
if err != nil {
return err
}
+9 -14
View File
@@ -1,7 +1,6 @@
package splitstore
import (
"sync"
"sync/atomic"
"time"
@@ -56,13 +55,12 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
if WarmupBoundary < epoch {
boundaryEpoch = epoch - WarmupBoundary
}
var mx sync.Mutex
batchHot := make([]blocks.Block, 0, batchSize)
count := new(int64)
xcount := new(int64)
missing := new(int64)
count := int64(0)
xcount := int64(0)
missing := int64(0)
visitor, err := s.markSetEnv.New("warmup", 0)
visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
if err != nil {
return xerrors.Errorf("error creating visitor: %w", err)
}
@@ -75,7 +73,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
return errStopWalk
}
atomic.AddInt64(count, 1)
count++
has, err := s.hot.Has(s.ctx, c)
if err != nil {
@@ -89,25 +87,22 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
blk, err := s.cold.Get(s.ctx, c)
if err != nil {
if err == bstore.ErrNotFound {
atomic.AddInt64(missing, 1)
missing++
return errStopWalk
}
return err
}
atomic.AddInt64(xcount, 1)
xcount++
mx.Lock()
batchHot = append(batchHot, blk)
if len(batchHot) == batchSize {
err = s.hot.PutMany(s.ctx, batchHot)
if err != nil {
mx.Unlock()
return err
}
batchHot = batchHot[:0]
}
mx.Unlock()
return nil
})
@@ -123,9 +118,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
}
}
log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)
log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
s.markSetSize = *count + *count>>2 // overestimate a bit
s.markSetSize = count + count>>2 // overestimate a bit
err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
if err != nil {
log.Warnf("error saving mark set size: %s", err)
+5 -33
View File
@@ -1,8 +1,6 @@
package splitstore
import (
"sync"
cid "github.com/ipfs/go-cid"
)
@@ -19,42 +17,16 @@ func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
return true, nil
}
type tmpVisitor struct {
type cidSetVisitor struct {
set *cid.Set
}
var _ ObjectVisitor = (*tmpVisitor)(nil)
func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
if isUnitaryObject(c) {
return false, nil
}
var _ ObjectVisitor = (*cidSetVisitor)(nil)
func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
return v.set.Visit(c), nil
}
func newTmpVisitor() ObjectVisitor {
return &tmpVisitor{set: cid.NewSet()}
}
type concurrentVisitor struct {
mx sync.Mutex
set *cid.Set
}
var _ ObjectVisitor = (*concurrentVisitor)(nil)
func newConcurrentVisitor() *concurrentVisitor {
return &concurrentVisitor{set: cid.NewSet()}
}
func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
if isUnitaryObject(c) {
return false, nil
}
v.mx.Lock()
defer v.mx.Unlock()
return v.set.Visit(c), nil
func tmpVisitor() ObjectVisitor {
return &cidSetVisitor{set: cid.NewSet()}
}
+2 -2
View File
@@ -1,2 +1,2 @@
/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBdRCBLUeKvoy22u5DcXs61adFn31v8WWCZgmBjDCjbsC
/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDUQJBA18njjXnG9RtLxoN3muvdU7PEy55QorUEsdAqdy
Binary file not shown.
Binary file not shown.
Binary file not shown.
-1
View File
@@ -90,7 +90,6 @@ func init() {
UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)
BuildType |= Build2k
+2 -1
View File
@@ -42,7 +42,8 @@ const UpgradeTurboHeight = -15
const UpgradeHyperdriveHeight = -16
const UpgradeChocolateHeight = -17
const UpgradeOhSnapHeight = 240
// 2022-01-17T19:00:00Z
const UpgradeOhSnapHeight = 30262
func init() {
policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
+1 -2
View File
@@ -54,8 +54,7 @@ const UpgradeHyperdriveHeight = 420
const UpgradeChocolateHeight = 312746
// 2022-02-10T19:23:00Z
const UpgradeOhSnapHeight = 682006
const UpgradeOhSnapHeight = 99999999
func init() {
policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
+1 -2
View File
@@ -67,8 +67,7 @@ const UpgradeHyperdriveHeight = 892800
// 2021-10-26T13:30:00Z
const UpgradeChocolateHeight = 1231620
// 2022-03-01T15:00:00Z
var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)
var UpgradeOhSnapHeight = abi.ChainEpoch(999999999999)
func init() {
if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
-11
View File
@@ -13,17 +13,6 @@ import (
func BlocksTopic(netName dtypes.NetworkName) string { return "/fil/blocks/" + string(netName) }
func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
func IndexerIngestTopic(netName dtypes.NetworkName) string {
nn := string(netName)
// The network name testnetnet is here for historical reasons.
// Going forward we aim to use the name `mainnet` where possible.
if nn == "testnetnet" {
nn = "mainnet"
}
return "/indexer/ingest/" + nn
}
func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
return protocol.ID("/fil/kad/" + string(netName))
}
+8 -8
View File
@@ -30,23 +30,23 @@
"sector_size": 2048
},
"v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.params": {
"cid": "QmNPc75iEfcahCwNKdqnWLtxnjspUGGR4iscjiz3wP3RtS",
"digest": "1b3cfd761a961543f9eb273e435a06a2",
"cid": "QmeNqDvsvyam4vqwCkstbxgb9S7RZEUeBDrJvBWKcpFKr6",
"digest": "532b53883ed4f794cb9d0db583d0df59",
"sector_size": 34359738368
},
"v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.vk": {
"cid": "QmdFFUe1gcz9MMHc6YW8aoV48w4ckvcERjt7PkydQAMfCN",
"digest": "3a6941983754737fde880d29c7094905",
"cid": "QmdLWr6moLUPScJZwoBckWqAeJkrBPAJPNLz8mWAfTdmXH",
"digest": "46990eb1bf5159c394a10309f269c1b6",
"sector_size": 34359738368
},
"v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.params": {
"cid": "QmUB6xTVjzBQGuDNeyJMrrJ1byk58vhPm8eY2Lv9pgwanp",
"digest": "1a392e7b759fb18e036c7559b5ece816",
"cid": "QmdQsi9uFhxK9cGwuK4rHuwKQoHkz6upYTCz4UdLiy1vA2",
"digest": "4223c63dbd94de1538006a14f37179e3",
"sector_size": 68719476736
},
"v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.vk": {
"cid": "Qmd794Jty7k26XJ8Eg4NDEks65Qk8G4GVfGkwqvymv8HAg",
"digest": "80e366df2f1011953c2d01c7b7c9ee8e",
"cid": "QmPirFX9wX99iMGA6zFY2CvcrdcDkj73X4MP6DLduvpbk9",
"digest": "ce39b614d788d3aef26bac1b28521d94",
"sector_size": 68719476736
},
"v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
+1 -1
View File
@@ -37,7 +37,7 @@ func BuildTypeString() string {
}
// BuildVersion is the local build version
const BuildVersion = "1.15.1-dev"
const BuildVersion = "1.14.0-rc3"
func UserVersion() string {
if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
@@ -16,7 +16,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
)
func init() {
@@ -63,11 +62,6 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
}
type RemoveDataCapProposal = verifreg{{.latestVersion}}.RemoveDataCapProposal
type RemoveDataCapRequest = verifreg{{.latestVersion}}.RemoveDataCapRequest
type RemoveDataCapParams = verifreg{{.latestVersion}}.RemoveDataCapParams
type RmDcProposalID = verifreg{{.latestVersion}}.RmDcProposalID
const SignatureDomainSeparation_RemoveDataCap = verifreg{{.latestVersion}}.SignatureDomainSeparation_RemoveDataCap
type State interface {
cbor.Marshaler
@@ -75,7 +69,6 @@ type State interface {
RootKey() (address.Address, error)
VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
GetState() interface{}
@@ -61,10 +61,6 @@ func (s *state{{.v}}) VerifierDataCap(addr address.Address) (bool, abi.StoragePo
return getDataCap(s.store, actors.Version{{.v}}, s.verifiers, addr)
}
func (s *state{{.v}}) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version{{.v}}, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state{{.v}}) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version{{.v}}, s.verifiers, cb)
}
@@ -81,11 +77,6 @@ func (s *state{{.v}}) verifiers() (adt.Map, error) {
return adt{{.v}}.AsMap(s.store, s.Verifiers{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
}
func (s *state{{.v}}) removeDataCapProposalIDs() (adt.Map, error) {
{{if le .v 6}}return nil, nil
{{else}}return adt{{.v}}.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin{{.v}}.DefaultHamtBitwidth){{end}}
}
func (s *state{{.v}}) GetState() interface{} {
return &s.State
}
-26
View File
@@ -6,7 +6,6 @@ import (
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
"golang.org/x/xerrors"
)
@@ -51,28 +50,3 @@ func forEachCap(store adt.Store, ver actors.Version, root rootFunc, cb func(addr
return cb(a, dcap)
})
}
func getRemoveDataCapProposalID(store adt.Store, ver actors.Version, root rootFunc, verifier address.Address, client address.Address) (bool, uint64, error) {
if verifier.Protocol() != address.ID {
return false, 0, xerrors.Errorf("can only look up ID addresses")
}
if client.Protocol() != address.ID {
return false, 0, xerrors.Errorf("can only look up ID addresses")
}
vh, err := root()
if err != nil {
return false, 0, xerrors.Errorf("loading verifreg: %w", err)
}
if vh == nil {
return false, 0, xerrors.Errorf("remove data cap proposal hamt not found. you are probably using an incompatible version of actors")
}
var id verifreg.RmDcProposalID
if found, err := vh.Get(abi.NewAddrPairKey(verifier, client), &id); err != nil {
return false, 0, xerrors.Errorf("looking up addr pair: %w", err)
} else if !found {
return false, 0, nil
}
return true, id.ProposalID, nil
}
-9
View File
@@ -53,10 +53,6 @@ func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version0, s.verifiers, addr)
}
func (s *state0) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version0, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version0, s.verifiers, cb)
}
@@ -73,11 +69,6 @@ func (s *state0) verifiers() (adt.Map, error) {
return adt0.AsMap(s.store, s.Verifiers)
}
func (s *state0) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state0) GetState() interface{} {
return &s.State
}
-9
View File
@@ -53,10 +53,6 @@ func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version2, s.verifiers, addr)
}
func (s *state2) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version2, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version2, s.verifiers, cb)
}
@@ -73,11 +69,6 @@ func (s *state2) verifiers() (adt.Map, error) {
return adt2.AsMap(s.store, s.Verifiers)
}
func (s *state2) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state2) GetState() interface{} {
return &s.State
}
-9
View File
@@ -54,10 +54,6 @@ func (s *state3) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version3, s.verifiers, addr)
}
func (s *state3) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version3, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state3) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version3, s.verifiers, cb)
}
@@ -74,11 +70,6 @@ func (s *state3) verifiers() (adt.Map, error) {
return adt3.AsMap(s.store, s.Verifiers, builtin3.DefaultHamtBitwidth)
}
func (s *state3) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state3) GetState() interface{} {
return &s.State
}
-9
View File
@@ -54,10 +54,6 @@ func (s *state4) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version4, s.verifiers, addr)
}
func (s *state4) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version4, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state4) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version4, s.verifiers, cb)
}
@@ -74,11 +70,6 @@ func (s *state4) verifiers() (adt.Map, error) {
return adt4.AsMap(s.store, s.Verifiers, builtin4.DefaultHamtBitwidth)
}
func (s *state4) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state4) GetState() interface{} {
return &s.State
}
-9
View File
@@ -54,10 +54,6 @@ func (s *state5) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version5, s.verifiers, addr)
}
func (s *state5) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version5, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state5) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version5, s.verifiers, cb)
}
@@ -74,11 +70,6 @@ func (s *state5) verifiers() (adt.Map, error) {
return adt5.AsMap(s.store, s.Verifiers, builtin5.DefaultHamtBitwidth)
}
func (s *state5) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state5) GetState() interface{} {
return &s.State
}
-9
View File
@@ -54,10 +54,6 @@ func (s *state6) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version6, s.verifiers, addr)
}
func (s *state6) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version6, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state6) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version6, s.verifiers, cb)
}
@@ -74,11 +70,6 @@ func (s *state6) verifiers() (adt.Map, error) {
return adt6.AsMap(s.store, s.Verifiers, builtin6.DefaultHamtBitwidth)
}
func (s *state6) removeDataCapProposalIDs() (adt.Map, error) {
return nil, nil
}
func (s *state6) GetState() interface{} {
return &s.State
}
-8
View File
@@ -54,10 +54,6 @@ func (s *state7) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
return getDataCap(s.store, actors.Version7, s.verifiers, addr)
}
func (s *state7) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
return getRemoveDataCapProposalID(s.store, actors.Version7, s.removeDataCapProposalIDs, verifier, client)
}
func (s *state7) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
return forEachCap(s.store, actors.Version7, s.verifiers, cb)
}
@@ -74,10 +70,6 @@ func (s *state7) verifiers() (adt.Map, error) {
return adt7.AsMap(s.store, s.Verifiers, builtin7.DefaultHamtBitwidth)
}
func (s *state7) removeDataCapProposalIDs() (adt.Map, error) {
return adt7.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin7.DefaultHamtBitwidth)
}
func (s *state7) GetState() interface{} {
return &s.State
}
@@ -27,7 +27,6 @@ import (
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/types"
verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
)
func init() {
@@ -152,20 +151,12 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
}
type RemoveDataCapProposal = verifreg7.RemoveDataCapProposal
type RemoveDataCapRequest = verifreg7.RemoveDataCapRequest
type RemoveDataCapParams = verifreg7.RemoveDataCapParams
type RmDcProposalID = verifreg7.RmDcProposalID
const SignatureDomainSeparation_RemoveDataCap = verifreg7.SignatureDomainSeparation_RemoveDataCap
type State interface {
cbor.Marshaler
RootKey() (address.Address, error)
VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
GetState() interface{}
+2 -2
View File
@@ -142,7 +142,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
go func() {
start := build.Clock.Now()
log.Debugw("start fetching randomness", "round", round)
log.Infow("start fetching randomness", "round", round)
resp, err := db.client.Get(ctx, round)
var br beacon.Response
@@ -152,7 +152,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
br.Entry.Round = resp.Round()
br.Entry.Data = resp.Signature()
}
log.Debugw("done fetching randomness", "round", round, "took", build.Clock.Since(start))
log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
out <- br
close(out)
}()
+32 -28
View File
@@ -32,7 +32,6 @@ import (
/* inline-gen end */
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -93,17 +92,16 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
partDone()
}()
ctx = blockstore.WithHotView(ctx)
makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (*vm.VM, error) {
makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) {
vmopt := &vm.VMOpts{
StateBase: base,
Epoch: e,
Epoch: epoch,
Rand: r,
Bstore: sm.ChainStore().StateBlockstore(),
Actors: NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, e),
NtwkVersion: sm.GetNtwkVersion,
BaseFee: baseFee,
LookbackState: stmgr.LookbackStateGetterForTipset(sm, ts),
}
@@ -111,7 +109,12 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
return sm.VMConstructor()(ctx, vmopt)
}
runCron := func(vmCron *vm.VM, epoch abi.ChainEpoch) error {
vmi, err := makeVmWithBaseState(pstate)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
}
runCron := func(epoch abi.ChainEpoch) error {
cronMsg := &types.Message{
To: cron.Address,
From: builtin.SystemActorAddr,
@@ -123,58 +126,59 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
Method: cron.Methods.EpochTick,
Params: nil,
}
ret, err := vmCron.ApplyImplicitMessage(ctx, cronMsg)
ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
if err != nil {
return xerrors.Errorf("running cron: %w", err)
return err
}
if em != nil {
if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
return xerrors.Errorf("callback failed on cron message: %w", err)
}
}
if ret.ExitCode != 0 {
return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
}
return nil
}
for i := parentEpoch; i < epoch; i++ {
var err error
if i > parentEpoch {
vmCron, err := makeVmWithBaseStateAndEpoch(pstate, i)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making cron vm: %w", err)
}
// run cron for null rounds if any
if err = runCron(vmCron, i); err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("running cron: %w", err)
if err := runCron(i); err != nil {
return cid.Undef, cid.Undef, err
}
pstate, err = vmCron.Flush(ctx)
pstate, err = vmi.Flush(ctx)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("flushing cron vm: %w", err)
return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
}
}
// handle state forks
// XXX: The state tree
pstate, err = sm.HandleStateForks(ctx, pstate, i, em, ts)
newState, err := sm.HandleStateForks(ctx, pstate, i, em, ts)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
}
if pstate != newState {
vmi, err = makeVmWithBaseState(newState)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
}
}
if err = vmi.SetBlockHeight(ctx, i+1); err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("error advancing vm an epoch: %w", err)
}
pstate = newState
}
partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyMessages)
vmi, err := makeVmWithBaseStateAndEpoch(pstate, epoch)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
}
var receipts []cbg.CBORMarshaler
processedMsgs := make(map[cid.Cid]struct{})
for _, b := range bms {
@@ -242,7 +246,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
partDone()
partDone = metrics.Timer(ctx, metrics.VMApplyCron)
if err := runCron(vmi, epoch); err != nil {
if err := runCron(epoch); err != nil {
return cid.Cid{}, cid.Cid{}, err
}
@@ -298,7 +302,7 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManag
parentEpoch = parent.Height
}
r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon(), sm.GetNetworkVersion)
r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon())
blkmsgs, err := sm.ChainStore().BlockMsgsForTipset(ctx, ts)
if err != nil {
+13 -19
View File
@@ -95,7 +95,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
return xerrors.Errorf("load parent tipset failed (%s): %w", h.Parents, err)
}
winPoStNv := filec.sm.GetNetworkVersion(ctx, baseTs.Height())
winPoStNv := filec.sm.GetNtwkVersion(ctx, baseTs.Height())
lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, filec.sm, baseTs, h.Height)
if err != nil {
@@ -182,7 +182,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
}
}
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", h.ParentStateRoot, stateroot)
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
}
if precp != h.ParentMessageReceipts {
@@ -458,7 +458,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
stateroot, _, err := filec.sm.TipSetState(ctx, baseTs)
if err != nil {
return xerrors.Errorf("failed to compute tipsettate for %s: %w", baseTs.Key(), err)
return err
}
st, err := state.LoadStateTree(filec.store.ActorStore(ctx), stateroot)
@@ -466,8 +466,8 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
return xerrors.Errorf("failed to load base state tree: %w", err)
}
nv := filec.sm.GetNetworkVersion(ctx, b.Header.Height)
pl := vm.PricelistByEpoch(b.Header.Height)
nv := filec.sm.GetNtwkVersion(ctx, b.Header.Height)
pl := vm.PricelistByEpoch(baseTs.Height())
var sumGasLimit int64
checkMsg := func(msg types.ChainMsg) error {
m := msg.VMMessage()
@@ -475,7 +475,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
// Phase 1: syntactic validation, as defined in the spec
minGas := pl.OnChainMessage(msg.ChainLength())
if err := m.ValidForBlockInclusion(minGas.Total(), nv); err != nil {
return xerrors.Errorf("msg %s invalid for block inclusion: %w", m.Cid(), err)
return err
}
// ValidForBlockInclusion checks if any single message does not exceed BlockGasLimit
@@ -488,10 +488,10 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
// Phase 2: (Partial) semantic validation:
// the sender exists and is an account actor, and the nonces make sense
var sender address.Address
if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version13 {
if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version13 {
sender, err = st.LookupID(m.From)
if err != nil {
return xerrors.Errorf("failed to lookup sender %s: %w", m.From, err)
return err
}
} else {
sender = m.From
@@ -541,7 +541,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
smArr := blockadt.MakeEmptyArray(tmpstore)
for i, m := range b.SecpkMessages {
if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version14 {
if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version14 {
if m.Signature.Type != crypto.SigTypeSecp256k1 {
return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err)
}
@@ -574,13 +574,12 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
bmroot, err := bmArr.Root()
if err != nil {
return xerrors.Errorf("failed to root bls msgs: %w", err)
return err
}
smroot, err := smArr.Root()
if err != nil {
return xerrors.Errorf("failed to root secp msgs: %w", err)
return err
}
mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{
@@ -588,7 +587,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
SecpkMessages: smroot,
})
if err != nil {
return xerrors.Errorf("failed to put msg meta: %w", err)
return err
}
if b.Header.Messages != mrcid {
@@ -596,12 +595,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
}
// Finally, flush.
err = vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
if err != nil {
return xerrors.Errorf("failed to flush:%w", err)
}
return nil
return vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
}
func (filec *FilecoinEC) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
+7 -2
View File
@@ -165,8 +165,13 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
Migration: UpgradeActorsV7,
PreMigrations: []stmgr.PreMigration{{
PreMigration: PreUpgradeActorsV7,
StartWithin: 180,
StartWithin: 120,
DontStartWithin: 60,
StopWithin: 35,
}, {
PreMigration: PreUpgradeActorsV7,
StartWithin: 30,
DontStartWithin: 15,
StopWithin: 5,
}},
Expensive: true,
@@ -1259,7 +1264,7 @@ func upgradeActorsV7Common(
root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
config nv15.Config,
) (cid.Cid, error) {
writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4)
writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB)
// TODO: pretty sure we'd achieve nothing by doing this, confirm in review
//buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), writeStore)
store := store.ActorStore(ctx, writeStore)
+4 -2
View File
@@ -491,8 +491,10 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, sys vm.Sysca
Actors: filcns.NewActorRegistry(),
Syscalls: mkFakedSigSyscalls(sys),
CircSupplyCalc: csc,
NetworkVersion: nv,
BaseFee: types.NewInt(0),
NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
return nv
},
BaseFee: types.NewInt(0),
}
vm, err := vm.NewVM(ctx, &vmopt)
if err != nil {
+24 -4
View File
@@ -94,8 +94,10 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
Actors: filcns.NewActorRegistry(),
Syscalls: mkFakedSigSyscalls(sys),
CircSupplyCalc: csc,
NetworkVersion: nv,
BaseFee: types.NewInt(0),
NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
return nv
},
BaseFee: types.NewInt(0),
}
vm, err := vm.NewVM(ctx, vmopt)
@@ -508,13 +510,31 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
// TODO: copied from actors test harness, deduplicate or remove from here
type fakeRand struct{}
func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (fr *fakeRand) GetChainRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
return out, nil
}
func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (fr *fakeRand) GetChainRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
return out, nil
}
func (fr *fakeRand) GetBeaconRandomnessV3(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
return out, nil
}
func (fr *fakeRand) GetBeaconRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
return out, nil
}
func (fr *fakeRand) GetBeaconRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
out := make([]byte, 32)
_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
return out, nil
+1 -1
View File
@@ -106,7 +106,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
curTs := mp.curTs
mp.curTsLk.Unlock()
epoch := curTs.Height() + 1
epoch := curTs.Height()
var baseFee big.Int
if len(curTs.Blocks()) > 0 {
-224
View File
@@ -1,224 +0,0 @@
//stm: #unit
package messagepool
import (
"context"
"fmt"
"testing"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
"github.com/stretchr/testify/assert"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/consensus/filcns"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/chain/wallet"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
)
func init() {
_ = logging.SetLogLevel("*", "INFO")
}
func getCheckMessageStatus(statusCode api.CheckStatusCode, msgStatuses []api.MessageCheckStatus) (*api.MessageCheckStatus, error) {
for i := 0; i < len(msgStatuses); i++ {
iMsgStatuses := msgStatuses[i]
if iMsgStatuses.CheckStatus.Code == statusCode {
return &iMsgStatuses, nil
}
}
return nil, fmt.Errorf("Could not find CheckStatusCode %s", statusCode)
}
func TestCheckMessages(t *testing.T) {
//stm: @CHAIN_MEMPOOL_CHECK_MESSAGES_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
if err != nil {
t.Fatal(err)
}
sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
tma.setBalance(sender, 1000e15)
target := mock.Address(1001)
var protos []*api.MessagePrototype
for i := 0; i < 5; i++ {
msg := &types.Message{
To: target,
From: sender,
Value: types.NewInt(1),
Nonce: uint64(i),
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 2<<10),
}
proto := &api.MessagePrototype{
Message: *msg,
ValidNonce: true,
}
protos = append(protos, proto)
}
messageStatuses, err := mp.CheckMessages(context.TODO(), protos)
assert.NoError(t, err)
for i := 0; i < len(messageStatuses); i++ {
iMsgStatuses := messageStatuses[i]
for j := 0; j < len(iMsgStatuses); j++ {
jStatus := iMsgStatuses[i]
assert.True(t, jStatus.OK)
}
}
}
func TestCheckPendingMessages(t *testing.T) {
//stm: @CHAIN_MEMPOOL_CHECK_PENDING_MESSAGES_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
if err != nil {
t.Fatal(err)
}
sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
tma.setBalance(sender, 1000e15)
target := mock.Address(1001)
// add a valid message to the pool
msg := &types.Message{
To: target,
From: sender,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 2<<10),
}
sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
mustAdd(t, mp, sm)
messageStatuses, err := mp.CheckPendingMessages(context.TODO(), sender)
assert.NoError(t, err)
for i := 0; i < len(messageStatuses); i++ {
iMsgStatuses := messageStatuses[i]
for j := 0; j < len(iMsgStatuses); j++ {
jStatus := iMsgStatuses[i]
assert.True(t, jStatus.OK)
}
}
}
func TestCheckReplaceMessages(t *testing.T) {
//stm: @CHAIN_MEMPOOL_CHECK_REPLACE_MESSAGES_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
if err != nil {
t.Fatal(err)
}
sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
tma.setBalance(sender, 1000e15)
target := mock.Address(1001)
// add a valid message to the pool
msg := &types.Message{
To: target,
From: sender,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 2<<10),
}
sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
mustAdd(t, mp, sm)
// create a new message with the same data, except that it is too big
var msgs []*types.Message
invalidmsg := &types.Message{
To: target,
From: sender,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 128<<10),
}
msgs = append(msgs, invalidmsg)
{
messageStatuses, err := mp.CheckReplaceMessages(context.TODO(), msgs)
if err != nil {
t.Fatal(err)
}
for i := 0; i < len(messageStatuses); i++ {
iMsgStatuses := messageStatuses[i]
status, err := getCheckMessageStatus(api.CheckStatusMessageSize, iMsgStatuses)
if err != nil {
t.Fatal(err)
}
// the replacement message should cause a status error
assert.False(t, status.OK)
}
}
}
+4 -25
View File
@@ -173,17 +173,10 @@ type MessagePool struct {
sigValCache *lru.TwoQueueCache
nonceCache *lru.Cache
evtTypes [3]journal.EventType
journal journal.Journal
}
type nonceCacheKey struct {
tsk types.TipSetKey
addr address.Address
}
type msgSet struct {
msgs map[uint64]*types.SignedMessage
nextNonce uint64
@@ -203,10 +196,10 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
return types.BigAdd(minPrice, types.NewInt(1))
}
func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.MessageSendSpec) {
func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSepc *api.MessageSendSpec) {
var maxFee abi.TokenAmount
if sendSpec != nil {
maxFee = sendSpec.MaxFee
if sendSepc != nil {
maxFee = sendSepc.MaxFee
}
if maxFee.Int == nil || maxFee.Equals(big.Zero()) {
mf, err := mff()
@@ -368,7 +361,6 @@ func (ms *msgSet) toSlice() []*types.SignedMessage {
func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) {
cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
noncecache, _ := lru.New(256)
cfg, err := loadConfig(ctx, ds)
if err != nil {
@@ -394,7 +386,6 @@ func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.Upgra
pruneCooldown: make(chan struct{}, 1),
blsSigCache: cache,
sigValCache: verifcache,
nonceCache: noncecache,
changes: lps.New(50),
localMsgs: namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
api: api,
@@ -628,7 +619,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
// For non local messages, if the message cannot be included in the next 20 blocks it returns
// a (soft) validation error.
func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
epoch := curTs.Height() + 1
epoch := curTs.Height()
minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {
@@ -1025,23 +1016,11 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address,
done := metrics.Timer(ctx, metrics.MpoolGetNonceDuration)
defer done()
nk := nonceCacheKey{
tsk: ts.Key(),
addr: addr,
}
n, ok := mp.nonceCache.Get(nk)
if ok {
return n.(uint64), nil
}
act, err := mp.api.GetActorAfter(addr, ts)
if err != nil {
return 0, err
}
mp.nonceCache.Add(nk, act.Nonce)
return act.Nonce, nil
}
-318
View File
@@ -1,4 +1,3 @@
//stm: #unit
package messagepool
import (
@@ -9,7 +8,6 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log/v2"
@@ -208,7 +206,6 @@ func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipS
func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
t.Helper()
//stm: @CHAIN_MEMPOOL_GET_NONCE_001
n, err := mp.GetNonce(context.TODO(), addr, types.EmptyTSK)
if err != nil {
t.Fatal(err)
@@ -227,8 +224,6 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) {
}
func TestMessagePool(t *testing.T) {
//stm: @CHAIN_MEMPOOL_GET_NONCE_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -330,7 +325,6 @@ func TestCheckMessageBig(t *testing.T) {
Message: *msg,
Signature: *sig,
}
//stm: @CHAIN_MEMPOOL_PUSH_001
err = mp.Add(context.TODO(), sm)
assert.ErrorIs(t, err, ErrMessageTooBig)
}
@@ -372,10 +366,8 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) {
tma.applyBlock(t, a)
tsa := mock.TipSet(a)
//stm: @CHAIN_MEMPOOL_PENDING_001
_, _ = mp.Pending(context.TODO())
//stm: @CHAIN_MEMPOOL_SELECT_001
selm, _ := mp.SelectMessages(context.Background(), tsa, 1)
if len(selm) == 0 {
t.Fatal("should have returned the rest of the messages")
@@ -436,7 +428,6 @@ func TestRevertMessages(t *testing.T) {
assertNonce(t, mp, sender, 4)
//stm: @CHAIN_MEMPOOL_PENDING_001
p, _ := mp.Pending(context.TODO())
fmt.Printf("%+v\n", p)
if len(p) != 3 {
@@ -495,7 +486,6 @@ func TestPruningSimple(t *testing.T) {
mp.Prune()
//stm: @CHAIN_MEMPOOL_PENDING_001
msgs, _ := mp.Pending(context.TODO())
if len(msgs) != 5 {
t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
@@ -538,7 +528,6 @@ func TestLoadLocal(t *testing.T) {
msgs := make(map[cid.Cid]struct{})
for i := 0; i < 10; i++ {
m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
//stm: @CHAIN_MEMPOOL_PUSH_001
cid, err := mp.Push(context.TODO(), m)
if err != nil {
t.Fatal(err)
@@ -555,7 +544,6 @@ func TestLoadLocal(t *testing.T) {
t.Fatal(err)
}
//stm: @CHAIN_MEMPOOL_PENDING_001
pmsgs, _ := mp.Pending(context.TODO())
if len(msgs) != len(pmsgs) {
t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
@@ -611,7 +599,6 @@ func TestClearAll(t *testing.T) {
gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
for i := 0; i < 10; i++ {
m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
//stm: @CHAIN_MEMPOOL_PUSH_001
_, err := mp.Push(context.TODO(), m)
if err != nil {
t.Fatal(err)
@@ -623,10 +610,8 @@ func TestClearAll(t *testing.T) {
mustAdd(t, mp, m)
}
//stm: @CHAIN_MEMPOOL_CLEAR_001
mp.Clear(context.Background(), true)
//stm: @CHAIN_MEMPOOL_PENDING_001
pending, _ := mp.Pending(context.TODO())
if len(pending) > 0 {
t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
@@ -669,7 +654,6 @@ func TestClearNonLocal(t *testing.T) {
gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
for i := 0; i < 10; i++ {
m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
//stm: @CHAIN_MEMPOOL_PUSH_001
_, err := mp.Push(context.TODO(), m)
if err != nil {
t.Fatal(err)
@@ -681,10 +665,8 @@ func TestClearNonLocal(t *testing.T) {
mustAdd(t, mp, m)
}
//stm: @CHAIN_MEMPOOL_CLEAR_001
mp.Clear(context.Background(), false)
//stm: @CHAIN_MEMPOOL_PENDING_001
pending, _ := mp.Pending(context.TODO())
if len(pending) != 10 {
t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
@@ -742,7 +724,6 @@ func TestUpdates(t *testing.T) {
for i := 0; i < 10; i++ {
m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
//stm: @CHAIN_MEMPOOL_PUSH_001
_, err := mp.Push(context.TODO(), m)
if err != nil {
t.Fatal(err)
@@ -764,302 +745,3 @@ func TestUpdates(t *testing.T) {
t.Fatal("expected closed channel, but got an update instead")
}
}
func TestMessageBelowMinGasFee(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
// fee is just below minimum gas fee
fee := minimumBaseFee.Uint64() - 1
{
msg := &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(fee),
GasPremium: types.NewInt(1),
Params: make([]byte, 32<<10),
}
sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
err = mp.Add(context.TODO(), sm)
assert.ErrorIs(t, err, ErrGasFeeCapTooLow)
}
}
func TestMessageValueTooHigh(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
totalFil := types.TotalFilecoinInt
extra := types.NewInt(1)
value := types.BigAdd(totalFil, extra)
{
msg := &types.Message{
To: to,
From: from,
Value: value,
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 32<<10),
}
sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
err = mp.Add(context.TODO(), sm)
assert.Error(t, err)
}
}
func TestMessageSignatureInvalid(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
msg := &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(minimumBaseFee.Uint64()),
GasPremium: types.NewInt(1),
Params: make([]byte, 32<<10),
}
badSig := &crypto.Signature{
Type: crypto.SigTypeSecp256k1,
Data: make([]byte, 0),
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *badSig,
}
err = mp.Add(context.TODO(), sm)
assert.Error(t, err)
// assert.Contains(t, err.Error(), "invalid signature length")
assert.Error(t, err)
}
}
func TestAddMessageTwice(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
// create a valid messages
sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
mustAdd(t, mp, sm)
// try to add it twice
err = mp.Add(context.TODO(), sm)
// assert.Contains(t, err.Error(), "with nonce 0 already in mpool")
assert.Error(t, err)
}
}
func TestAddMessageTwiceNonceGap(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
// create message with invalid nonce (1)
sm := makeTestMessage(w, from, to, 1, 50_000_000, minimumBaseFee.Uint64())
mustAdd(t, mp, sm)
// then try to add message again
err = mp.Add(context.TODO(), sm)
// assert.Contains(t, err.Error(), "unfulfilled nonce gap")
assert.Error(t, err)
}
}
func TestAddMessageTwiceCidDiff(t *testing.T) {
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
mustAdd(t, mp, sm)
// Create message with different data, so CID is different
sm2 := makeTestMessage(w, from, to, 0, 50_000_001, minimumBaseFee.Uint64())
//stm: @CHAIN_MEMPOOL_PUSH_001
// then try to add message again
err = mp.Add(context.TODO(), sm2)
// assert.Contains(t, err.Error(), "replace by fee has too low GasPremium")
assert.Error(t, err)
}
}
func TestAddMessageTwiceCidDiffReplaced(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
mustAdd(t, mp, sm)
// Create message with different data, so CID is different
sm2 := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64()*2)
mustAdd(t, mp, sm2)
}
}
func TestRemoveMessage(t *testing.T) {
//stm: @CHAIN_MEMPOOL_PUSH_001
tma := newTestMpoolAPI()
w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)
from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)
tma.setBalance(from, 1000e9)
ds := datastore.NewMapDatastore()
mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
assert.NoError(t, err)
to := mock.Address(1001)
{
sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
mustAdd(t, mp, sm)
//stm: @CHAIN_MEMPOOL_REMOVE_001
// remove message for sender
mp.Remove(context.TODO(), from, sm.Message.Nonce, true)
//stm: @CHAIN_MEMPOOL_PENDING_FOR_001
// check messages in pool: should be none present
msgs := mp.pendingFor(context.TODO(), from)
assert.Len(t, msgs, 0)
}
}
-3
View File
@@ -1,4 +1,3 @@
//stm: #unit
package messagepool
import (
@@ -17,7 +16,6 @@ import (
)
func TestRepubMessages(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001
oldRepublishBatchDelay := RepublishBatchDelay
RepublishBatchDelay = time.Microsecond
defer func() {
@@ -59,7 +57,6 @@ func TestRepubMessages(t *testing.T) {
for i := 0; i < 10; i++ {
m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
//stm: @CHAIN_MEMPOOL_PUSH_001
_, err := mp.Push(context.TODO(), m)
if err != nil {
t.Fatal(err)
-26
View File
@@ -1,4 +1,3 @@
//stm: #unit
package messagepool
import (
@@ -75,8 +74,6 @@ func makeTestMpool() (*MessagePool, *testMpoolAPI) {
}
func TestMessageChains(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001
//stm: @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
mp, tma := makeTestMpool()
// the actors
@@ -313,8 +310,6 @@ func TestMessageChains(t *testing.T) {
}
func TestMessageChainSkipping(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
// regression test for chain skip bug
mp, tma := makeTestMpool()
@@ -387,7 +382,6 @@ func TestMessageChainSkipping(t *testing.T) {
}
func TestBasicMessageSelection(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
oldMaxNonceGap := MaxNonceGap
MaxNonceGap = 1000
defer func() {
@@ -538,7 +532,6 @@ func TestBasicMessageSelection(t *testing.T) {
}
func TestMessageSelectionTrimmingGas(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -602,7 +595,6 @@ func TestMessageSelectionTrimmingGas(t *testing.T) {
}
func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -649,7 +641,6 @@ func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
}
func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -716,7 +707,6 @@ func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
}
func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -798,7 +788,6 @@ func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
}
func TestPriorityMessageSelection(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -878,7 +867,6 @@ func TestPriorityMessageSelection(t *testing.T) {
}
func TestPriorityMessageSelection2(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -946,7 +934,6 @@ func TestPriorityMessageSelection2(t *testing.T) {
}
func TestPriorityMessageSelection3(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
mp, tma := makeTestMpool()
// the actors
@@ -1041,8 +1028,6 @@ func TestPriorityMessageSelection3(t *testing.T) {
}
func TestOptimalMessageSelection1(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
// this test uses just a single actor sending messages with a low tq
// the chain depenent merging algorithm should pick messages from the actor
// from the start
@@ -1109,8 +1094,6 @@ func TestOptimalMessageSelection1(t *testing.T) {
}
func TestOptimalMessageSelection2(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
// this test uses two actors sending messages to each other, with the first
// actor paying (much) higher gas premium than the second.
// We select with a low ticket quality; the chain depenent merging algorithm should pick
@@ -1190,8 +1173,6 @@ func TestOptimalMessageSelection2(t *testing.T) {
}
func TestOptimalMessageSelection3(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
// this test uses 10 actors sending a block of messages to each other, with the the first
// actors paying higher gas premium than the subsequent actors.
// We select with a low ticket quality; the chain dependent merging algorithm should pick
@@ -1435,8 +1416,6 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
}
func TestCompetitiveMessageSelectionExp(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
if testing.Short() {
t.Skip("skipping in short mode")
}
@@ -1460,8 +1439,6 @@ func TestCompetitiveMessageSelectionExp(t *testing.T) {
}
func TestCompetitiveMessageSelectionZipf(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
if testing.Short() {
t.Skip("skipping in short mode")
}
@@ -1485,7 +1462,6 @@ func TestCompetitiveMessageSelectionZipf(t *testing.T) {
}
func TestGasReward(t *testing.T) {
//stm: @CHAIN_MEMPOOL_GET_GAS_REWARD_001
tests := []struct {
Premium uint64
FeeCap uint64
@@ -1518,8 +1494,6 @@ func TestGasReward(t *testing.T) {
}
func TestRealWorldSelection(t *testing.T) {
//stm: @TOKEN_WALLET_NEW_001, @TOKEN_WALLET_SIGN_001, @CHAIN_MEMPOOL_SELECT_001
// load test-messages.json.gz and rewrite the messages so that
// 1) we map each real actor to a test actor so that we can sign the messages
// 2) adjust the nonces so that they start from 0
@@ -1,4 +1,3 @@
//stm: #unit
package messagesigner
import (
@@ -61,7 +60,6 @@ func TestMessageSignerSignMessage(t *testing.T) {
to2, err := w.WalletNew(ctx, types.KTSecp256k1)
require.NoError(t, err)
//stm: @CHAIN_MESSAGE_SIGNER_NEW_SIGNER_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_005
type msgSpec struct {
msg *types.Message
mpoolNonce [1]uint64
+22 -40
View File
@@ -4,8 +4,6 @@ import (
"context"
"encoding/binary"
"github.com/filecoin-project/go-state-types/network"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/chain/beacon"
@@ -72,7 +70,7 @@ func (sr *stateRand) GetBeaconRandomnessTipset(ctx context.Context, round abi.Ch
return randTs, nil
}
func (sr *stateRand) getChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte, lookback bool) ([]byte, error) {
func (sr *stateRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte, lookback bool) ([]byte, error) {
_, span := trace.StartSpan(ctx, "store.GetChainRandomness")
defer span.End()
span.AddAttributes(trace.Int64Attribute("round", int64(round)))
@@ -103,26 +101,32 @@ func (sr *stateRand) getChainRandomness(ctx context.Context, pers crypto.DomainS
return DrawRandomness(mtb.Ticket.VRFProof, pers, round, entropy)
}
type NetworkVersionGetter func(context.Context, abi.ChainEpoch) network.Version
type stateRand struct {
cs *store.ChainStore
blks []cid.Cid
beacon beacon.Schedule
networkVersionGetter NetworkVersionGetter
cs *store.ChainStore
blks []cid.Cid
beacon beacon.Schedule
}
func NewStateRand(cs *store.ChainStore, blks []cid.Cid, b beacon.Schedule, networkVersionGetter NetworkVersionGetter) vm.Rand {
func NewStateRand(cs *store.ChainStore, blks []cid.Cid, b beacon.Schedule) vm.Rand {
return &stateRand{
cs: cs,
blks: blks,
beacon: b,
networkVersionGetter: networkVersionGetter,
cs: cs,
blks: blks,
beacon: b,
}
}
// network v0-12
func (sr *stateRand) getBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (sr *stateRand) GetChainRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
return sr.GetChainRandomness(ctx, pers, round, entropy, true)
}
// network v13 and on
func (sr *stateRand) GetChainRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
return sr.GetChainRandomness(ctx, pers, round, entropy, false)
}
// network v0-12
func (sr *stateRand) GetBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, true)
if err != nil {
return nil, err
@@ -139,7 +143,7 @@ func (sr *stateRand) getBeaconRandomnessV1(ctx context.Context, pers crypto.Doma
}
// network v13
func (sr *stateRand) getBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (sr *stateRand) GetBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, false)
if err != nil {
return nil, err
@@ -156,9 +160,9 @@ func (sr *stateRand) getBeaconRandomnessV2(ctx context.Context, pers crypto.Doma
}
// network v14 and on
func (sr *stateRand) getBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
func (sr *stateRand) GetBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
if filecoinEpoch < 0 {
return sr.getBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
return sr.GetBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
}
be, err := sr.extractBeaconEntryForEpoch(ctx, filecoinEpoch)
@@ -170,28 +174,6 @@ func (sr *stateRand) getBeaconRandomnessV3(ctx context.Context, pers crypto.Doma
return DrawRandomness(be.Data, pers, filecoinEpoch, entropy)
}
func (sr *stateRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
nv := sr.networkVersionGetter(ctx, filecoinEpoch)
if nv >= network.Version13 {
return sr.getChainRandomness(ctx, pers, filecoinEpoch, entropy, false)
}
return sr.getChainRandomness(ctx, pers, filecoinEpoch, entropy, true)
}
func (sr *stateRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
nv := sr.networkVersionGetter(ctx, filecoinEpoch)
if nv >= network.Version14 {
return sr.getBeaconRandomnessV3(ctx, pers, filecoinEpoch, entropy)
} else if nv == network.Version13 {
return sr.getBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
} else {
return sr.getBeaconRandomnessV1(ctx, pers, filecoinEpoch, entropy)
}
}
func (sr *stateRand) extractBeaconEntryForEpoch(ctx context.Context, filecoinEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
randTs, err := sr.GetBeaconRandomnessTipset(ctx, filecoinEpoch, false)
if err != nil {
-10
View File
@@ -1,4 +1,3 @@
//stm:#unit
package rand_test
import (
@@ -56,13 +55,11 @@ func TestNullRandomnessV1(t *testing.T) {
randEpoch := ts.TipSet.TipSet().Height() - 2
//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V1_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01, @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_TIPSET_02
rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
if err != nil {
t.Fatal(err)
}
//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(beforeNullHeight)+offset)
select {
@@ -71,7 +68,6 @@ func TestNullRandomnessV1(t *testing.T) {
t.Fatal(resp.Err)
}
//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01
rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
if err != nil {
t.Fatal(err)
@@ -135,13 +131,11 @@ func TestNullRandomnessV2(t *testing.T) {
randEpoch := ts.TipSet.TipSet().Height() - 2
//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V2_01
rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
if err != nil {
t.Fatal(err)
}
//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(ts.TipSet.TipSet().Height())+offset)
select {
@@ -150,7 +144,6 @@ func TestNullRandomnessV2(t *testing.T) {
t.Fatal(resp.Err)
}
//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01, @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_TIPSET_03
// note that the randEpoch passed to DrawRandomness is still randEpoch (not the latest ts height)
rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
if err != nil {
@@ -219,13 +212,11 @@ func TestNullRandomnessV3(t *testing.T) {
randEpoch := ts.TipSet.TipSet().Height() - 2
//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V3_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01
rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
if err != nil {
t.Fatal(err)
}
//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(randEpoch)+offset)
select {
@@ -234,7 +225,6 @@ func TestNullRandomnessV3(t *testing.T) {
t.Fatal(resp.Err)
}
//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01
rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
if err != nil {
t.Fatal(err)
+2 -2
View File
@@ -359,7 +359,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
return nil, xerrors.Errorf("failed to get randomness for winning post: %w", err)
}
nv := sm.GetNetworkVersion(ctx, ts.Height())
nv := sm.GetNtwkVersion(ctx, ts.Height())
sectors, err := GetSectorsForWinningPoSt(ctx, nv, pv, sm, lbst, maddr, prand)
if err != nil {
@@ -421,7 +421,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
hmp, err := minerHasMinPower(ctx, sm, addr, lookbackTs)
// TODO: We're blurring the lines between a "runtime network version" and a "Lotus upgrade epoch", is that unavoidable?
if sm.GetNetworkVersion(ctx, baseTs.Height()) <= network.Version3 {
if sm.GetNtwkVersion(ctx, baseTs.Height()) <= network.Version3 {
return hmp, err
}
+4 -4
View File
@@ -75,12 +75,12 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
vmopt := &vm.VMOpts{
StateBase: bstate,
Epoch: pheight + 1,
Rand: rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion),
Rand: rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon),
Bstore: sm.cs.StateBlockstore(),
Actors: sm.tsExec.NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, pheight+1),
NtwkVersion: sm.GetNtwkVersion,
BaseFee: types.NewInt(0),
LookbackState: LookbackStateGetterForTipset(sm, ts),
}
@@ -186,7 +186,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
return nil, fmt.Errorf("failed to handle fork: %w", err)
}
r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)
r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon)
if span.IsRecordingEvents() {
span.AddAttributes(
@@ -204,7 +204,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
Actors: sm.tsExec.NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, ts.Height()+1),
NtwkVersion: sm.GetNtwkVersion,
BaseFee: ts.Blocks()[0].ParentBaseFee,
LookbackState: LookbackStateGetterForTipset(sm, ts),
}
+17 -5
View File
@@ -357,7 +357,7 @@ func (sm *StateManager) VMConstructor() func(context.Context, *vm.VMOpts) (*vm.V
}
}
func (sm *StateManager) GetNetworkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
// The epochs here are the _last_ epoch for every version, or -1 if the
// version is disabled.
for _, spec := range sm.networkVersions {
@@ -378,9 +378,16 @@ func (sm *StateManager) GetRandomnessFromBeacon(ctx context.Context, personaliza
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
}
r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)
r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon)
rnv := sm.GetNtwkVersion(ctx, randEpoch)
return r.GetBeaconRandomness(ctx, personalization, randEpoch, entropy)
if rnv >= network.Version14 {
return r.GetBeaconRandomnessV3(ctx, personalization, randEpoch, entropy)
} else if rnv == network.Version13 {
return r.GetBeaconRandomnessV2(ctx, personalization, randEpoch, entropy)
}
return r.GetBeaconRandomnessV1(ctx, personalization, randEpoch, entropy)
}
@@ -390,7 +397,12 @@ func (sm *StateManager) GetRandomnessFromTickets(ctx context.Context, personaliz
return nil, xerrors.Errorf("loading tipset key: %w", err)
}
r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)
r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon)
rnv := sm.GetNtwkVersion(ctx, randEpoch)
return r.GetChainRandomness(ctx, personalization, randEpoch, entropy)
if rnv >= network.Version13 {
return r.GetChainRandomnessV2(ctx, personalization, randEpoch, entropy)
}
return r.GetChainRandomnessV1(ctx, personalization, randEpoch, entropy)
}
+3 -3
View File
@@ -79,7 +79,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
// future. It's not guaranteed to be accurate... but that's fine.
}
r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)
r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon)
vmopt := &vm.VMOpts{
StateBase: base,
Epoch: height,
@@ -88,7 +88,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
Actors: sm.tsExec.NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, height),
NtwkVersion: sm.GetNtwkVersion,
BaseFee: ts.Blocks()[0].ParentBaseFee,
LookbackState: LookbackStateGetterForTipset(sm, ts),
}
@@ -128,7 +128,7 @@ func LookbackStateGetterForTipset(sm *StateManager, ts *types.TipSet) vm.Lookbac
func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, cid.Cid, error) {
var lbr abi.ChainEpoch
lb := policy.GetWinningPoStSectorSetLookback(sm.GetNetworkVersion(ctx, round))
lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round))
if round > lb {
lbr = round - lb
}
+4 -174
View File
@@ -1,24 +1,19 @@
package sub
import (
"bytes"
"context"
"encoding/binary"
"sync"
"fmt"
"time"
address "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-legs/dtsync"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/consensus"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/sub/ratelimit"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/node/impl/client"
"github.com/filecoin-project/lotus/node/impl/full"
lru "github.com/hashicorp/golang-lru"
blocks "github.com/ipfs/go-block-format"
bserv "github.com/ipfs/go-blockservice"
@@ -173,12 +168,12 @@ func fetchCids(
cidIndex := make(map[cid.Cid]int)
for i, c := range cids {
if c.Prefix() != msgCidPrefix {
return xerrors.Errorf("invalid msg CID: %s", c)
return fmt.Errorf("invalid msg CID: %s", c)
}
cidIndex[c] = i
}
if len(cids) != len(cidIndex) {
return xerrors.Errorf("duplicate CIDs in fetchCids input")
return fmt.Errorf("duplicate CIDs in fetchCids input")
}
for block := range bserv.GetBlocks(ctx, cids) {
@@ -201,7 +196,7 @@ func fetchCids(
if len(cidIndex) > 0 {
err := ctx.Err()
if err == nil {
err = xerrors.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
}
return err
}
@@ -449,168 +444,3 @@ func recordFailure(ctx context.Context, metric *stats.Int64Measure, failureType
)
stats.Record(ctx, metric.M(1))
}
type peerMsgInfo struct {
peerID peer.ID
lastCid cid.Cid
lastSeqno uint64
rateLimit *ratelimit.Window
mutex sync.Mutex
}
type IndexerMessageValidator struct {
self peer.ID
peerCache *lru.TwoQueueCache
chainApi full.ChainModuleAPI
stateApi full.StateModuleAPI
}
func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator {
peerCache, _ := lru.New2Q(8192)
return &IndexerMessageValidator{
self: self,
peerCache: peerCache,
chainApi: chainApi,
stateApi: stateApi,
}
}
func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
// This chain-node should not be publishing its own messages. These are
// relayed from market-nodes. If a node appears to be local, reject it.
if pid == v.self {
log.Debug("ignoring indexer message from self")
stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
originPeer := msg.GetFrom()
if originPeer == v.self {
log.Debug("ignoring indexer message originating from self")
stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
return pubsub.ValidationIgnore
}
idxrMsg := dtsync.Message{}
err := idxrMsg.UnmarshalCBOR(bytes.NewBuffer(msg.Data))
if err != nil {
log.Errorw("Could not decode indexer pubsub message", "err", err)
return pubsub.ValidationReject
}
if len(idxrMsg.ExtraData) == 0 {
log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
return pubsub.ValidationIgnore
}
// Get miner info from lotus
minerAddr, err := address.NewFromBytes(idxrMsg.ExtraData)
if err != nil {
log.Warnw("cannot parse extra data as miner address", "err", err, "extraData", idxrMsg.ExtraData)
return pubsub.ValidationReject
}
minerID := minerAddr.String()
msgCid := idxrMsg.Cid
var msgInfo *peerMsgInfo
val, ok := v.peerCache.Get(minerID)
if !ok {
msgInfo = &peerMsgInfo{}
} else {
msgInfo = val.(*peerMsgInfo)
}
// Lock this peer's message info.
msgInfo.mutex.Lock()
defer msgInfo.mutex.Unlock()
if ok {
// Reject replayed messages.
seqno := binary.BigEndian.Uint64(msg.Message.GetSeqno())
if seqno <= msgInfo.lastSeqno {
log.Debugf("ignoring replayed indexer message")
return pubsub.ValidationIgnore
}
msgInfo.lastSeqno = seqno
}
if !ok || originPeer != msgInfo.peerID {
// Check that the miner ID maps to the peer that sent the message.
err = v.authenticateMessage(ctx, minerAddr, originPeer)
if err != nil {
log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID)
stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
return pubsub.ValidationReject
}
msgInfo.peerID = originPeer
if !ok {
// Add msgInfo to cache only after being authenticated. If two
// messages from the same peer are handled concurrently, there is a
// small chance that one msgInfo could replace the other here when
// the info is first cached. This is OK, so no need to prevent it.
v.peerCache.Add(minerID, msgInfo)
}
}
// See if message needs to be ignored due to rate limiting.
if v.rateLimitPeer(msgInfo, msgCid) {
return pubsub.ValidationIgnore
}
stats.Record(ctx, metrics.IndexerMessageValidationSuccess.M(1))
return pubsub.ValidationAccept
}
func (v *IndexerMessageValidator) rateLimitPeer(msgInfo *peerMsgInfo, msgCid cid.Cid) bool {
const (
msgLimit = 5
msgTimeLimit = 10 * time.Second
repeatTimeLimit = 2 * time.Hour
)
timeWindow := msgInfo.rateLimit
// Check overall message rate.
if timeWindow == nil {
timeWindow = ratelimit.NewWindow(msgLimit, msgTimeLimit)
msgInfo.rateLimit = timeWindow
} else if msgInfo.lastCid == msgCid {
// Check if this is a repeat of the previous message data.
if time.Since(timeWindow.Newest()) < repeatTimeLimit {
log.Warnw("ignoring repeated indexer message", "sender", msgInfo.peerID)
return true
}
}
err := timeWindow.Add()
if err != nil {
log.Warnw("ignoring indexer message", "sender", msgInfo.peerID, "err", err)
return true
}
msgInfo.lastCid = msgCid
return false
}
func (v *IndexerMessageValidator) authenticateMessage(ctx context.Context, minerAddress address.Address, peerID peer.ID) error {
ts, err := v.chainApi.ChainHead(ctx)
if err != nil {
return err
}
minerInfo, err := v.stateApi.StateMinerInfo(ctx, minerAddress, ts.Key())
if err != nil {
return err
}
if minerInfo.PeerId == nil {
return xerrors.New("no peer id for miner")
}
if *minerInfo.PeerId != peerID {
return xerrors.New("miner id does not map to peer that sent message")
}
return nil
}
+1 -72
View File
@@ -1,21 +1,13 @@
//stm: #unit
package sub
import (
"bytes"
"context"
"testing"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-legs/dtsync"
"github.com/filecoin-project/lotus/api/mocks"
address "github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/golang/mock/gomock"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
pubsub "github.com/libp2p/go-libp2p-pubsub"
pb "github.com/libp2p/go-libp2p-pubsub/pb"
)
type getter struct {
@@ -57,7 +49,6 @@ func TestFetchCidsWithDedup(t *testing.T) {
}
g := &getter{msgs}
//stm: @CHAIN_INCOMING_FETCH_MESSAGES_BY_CID_001
// the cids have a duplicate
res, err := FetchMessagesByCids(context.TODO(), g, append(cids, cids[0]))
@@ -70,65 +61,3 @@ func TestFetchCidsWithDedup(t *testing.T) {
t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1])
}
}
func TestIndexerMessageValidator_Validate(t *testing.T) {
validCid, err := cid.Decode("QmbpDgg5kRLDgMxS8vPKNFXEcA6D5MC4CkuUdSWDVtHPGK")
if err != nil {
t.Fatal(err)
}
tests := []struct {
name string
selfPID string
senderPID string
extraData []byte
wantValidation pubsub.ValidationResult
}{
{
name: "invalid extra data is rejected",
selfPID: "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
senderPID: "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ",
extraData: []byte("f0127896"), // note, casting encoded address to byte is invalid.
wantValidation: pubsub.ValidationReject,
},
{
name: "same sender and receiver is ignored",
selfPID: "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
senderPID: "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
wantValidation: pubsub.ValidationIgnore,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
mc := gomock.NewController(t)
node := mocks.NewMockFullNode(mc)
subject := NewIndexerMessageValidator(peer.ID(tc.selfPID), node, node)
message := dtsync.Message{
Cid: validCid,
Addrs: nil,
ExtraData: tc.extraData,
}
buf := bytes.NewBuffer(nil)
if err := message.MarshalCBOR(buf); err != nil {
t.Fatal(err)
}
topic := "topic"
pbm := &pb.Message{
Data: buf.Bytes(),
Topic: &topic,
From: nil,
Seqno: nil,
}
validate := subject.Validate(context.Background(), peer.ID(tc.senderPID), &pubsub.Message{
Message: pbm,
ReceivedFrom: peer.ID(tc.senderPID),
ValidatorData: nil,
})
if validate != tc.wantValidation {
t.Fatalf("expected %v but got %v", tc.wantValidation, validate)
}
})
}
}
-89
View File
@@ -1,89 +0,0 @@
package ratelimit
import "errors"
var ErrRateLimitExceeded = errors.New("rate limit exceeded")
type queue struct {
buf []int64
count int
head int
tail int
}
// cap returns the queue capacity
func (q *queue) cap() int {
return len(q.buf)
}
// len returns the number of items in the queue
func (q *queue) len() int {
return q.count
}
// push adds an element to the end of the queue.
func (q *queue) push(elem int64) error {
if q.count == len(q.buf) {
return ErrRateLimitExceeded
}
q.buf[q.tail] = elem
// Calculate new tail position.
q.tail = q.next(q.tail)
q.count++
return nil
}
// pop removes and returns the element from the front of the queue.
func (q *queue) pop() int64 {
if q.count <= 0 {
panic("pop from empty queue")
}
ret := q.buf[q.head]
// Calculate new head position.
q.head = q.next(q.head)
q.count--
return ret
}
// front returns the element at the front of the queue. This is the element
// that would be returned by pop(). This call panics if the queue is empty.
func (q *queue) front() int64 {
if q.count <= 0 {
panic("front() called when empty")
}
return q.buf[q.head]
}
// back returns the element at the back of the queue. This call panics if the
// queue is empty.
func (q *queue) back() int64 {
if q.count <= 0 {
panic("back() called when empty")
}
return q.buf[q.prev(q.tail)]
}
// prev returns the previous buffer position wrapping around buffer.
func (q *queue) prev(i int) int {
if i == 0 {
return len(q.buf) - 1
}
return (i - 1) % len(q.buf)
}
// next returns the next buffer position wrapping around buffer.
func (q *queue) next(i int) int {
return (i + 1) % len(q.buf)
}
// truncate pops values that are less than or equal the specified threshold.
func (q *queue) truncate(threshold int64) {
for q.count != 0 && q.buf[q.head] <= threshold {
// pop() without returning a value
q.head = q.next(q.head)
q.count--
}
}
-70
View File
@@ -1,70 +0,0 @@
package ratelimit
import "time"
// Window is a time windows for counting events within a span of time. The
// windows slides forward in time so that it spans from the most recent event
// to size time in the past.
type Window struct {
q *queue
size int64
}
// NewWindow creates a new Window that limits the number of events to maximum
// count of events within a duration of time. The capacity sets the maximum
// number of events, and size sets the span of time over which the events are
// counted.
func NewWindow(capacity int, size time.Duration) *Window {
return &Window{
q: &queue{
buf: make([]int64, capacity),
},
size: int64(size),
}
}
// Add attempts to append a new timestamp into the current window. Previously
// added values that are not not within `size` difference from the value being
// added are first removed. Add fails if adding the value would cause the
// window to exceed capacity.
func (w *Window) Add() error {
now := time.Now().UnixNano()
if w.Len() != 0 {
w.q.truncate(now - w.size)
}
return w.q.push(now)
}
// Cap returns the maximum number of items the window can hold.
func (w *Window) Cap() int {
return w.q.cap()
}
// Len returns the number of elements currently in the window.
func (w *Window) Len() int {
return w.q.len()
}
// Span returns the distance from the first to the last item in the window.
func (w *Window) Span() time.Duration {
if w.q.len() < 2 {
return 0
}
return time.Duration(w.q.back() - w.q.front())
}
// Oldest returns the oldest timestamp in the window.
func (w *Window) Oldest() time.Time {
if w.q.len() == 0 {
return time.Time{}
}
return time.Unix(0, w.q.front())
}
// Newest returns the newest timestamp in the window.
func (w *Window) Newest() time.Time {
if w.q.len() == 0 {
return time.Time{}
}
return time.Unix(0, w.q.back())
}
-61
View File
@@ -1,61 +0,0 @@
package ratelimit
import (
"testing"
"time"
)
func TestWindow(t *testing.T) {
const (
maxEvents = 3
timeLimit = 100 * time.Millisecond
)
w := NewWindow(maxEvents, timeLimit)
if w.Len() != 0 {
t.Fatal("q.Len() =", w.Len(), "expect 0")
}
if w.Cap() != maxEvents {
t.Fatal("q.Cap() =", w.Cap(), "expect 3")
}
if !w.Newest().IsZero() {
t.Fatal("expected newest to be zero time with empty window")
}
if !w.Oldest().IsZero() {
t.Fatal("expected oldest to be zero time with empty window")
}
if w.Span() != 0 {
t.Fatal("expected span to be zero time with empty window")
}
var err error
for i := 0; i < maxEvents; i++ {
err = w.Add()
if err != nil {
t.Fatalf("cannot add event %d", i)
}
}
if w.Len() != maxEvents {
t.Fatalf("q.Len() is %d, expected %d", w.Len(), maxEvents)
}
if err = w.Add(); err != ErrRateLimitExceeded {
t.Fatalf("add event %d within time limit should have failed with err: %s", maxEvents+1, ErrRateLimitExceeded)
}
time.Sleep(timeLimit)
if err = w.Add(); err != nil {
t.Fatalf("cannot add event after time limit: %s", err)
}
prev := w.Newest()
time.Sleep(timeLimit)
err = w.Add()
if err != nil {
t.Fatalf("cannot add event")
}
if w.Newest().Before(prev) {
t.Fatal("newest is before previous value")
}
if w.Oldest().Before(prev) {
t.Fatal("oldest is before previous value")
}
}
+2 -45
View File
@@ -1,4 +1,3 @@
//stm: #unit
package chain_test
import (
@@ -104,7 +103,7 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil {
ctx: ctx,
cancel: cancel,
mn: mocknet.New(),
mn: mocknet.New(ctx),
g: g,
us: filcns.DefaultUpgradeSchedule(),
}
@@ -158,7 +157,7 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn
ctx: ctx,
cancel: cancel,
mn: mocknet.New(),
mn: mocknet.New(ctx),
g: g,
us: sched,
}
@@ -463,8 +462,6 @@ func (tu *syncTestUtil) waitUntilSyncTarget(to int, target *types.TipSet) {
}
func TestSyncSimple(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 50
tu := prepSyncTest(t, H)
@@ -481,8 +478,6 @@ func TestSyncSimple(t *testing.T) {
}
func TestSyncMining(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 50
tu := prepSyncTest(t, H)
@@ -505,8 +500,6 @@ func TestSyncMining(t *testing.T) {
}
func TestSyncBadTimestamp(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 50
tu := prepSyncTest(t, H)
@@ -561,8 +554,6 @@ func (wpp badWpp) ComputeProof(context.Context, []proof7.ExtendedSectorInfo, abi
}
func TestSyncBadWinningPoSt(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 15
tu := prepSyncTest(t, H)
@@ -592,9 +583,6 @@ func (tu *syncTestUtil) loadChainToNode(to int) {
}
func TestSyncFork(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -662,9 +650,6 @@ func TestSyncFork(t *testing.T) {
// A and B both include _different_ messages from sender X with nonce N (where N is the correct nonce for X).
// We can confirm that the state can be correctly computed, and that `MessagesForTipset` behaves as expected.
func TestDuplicateNonce(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001
//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -719,7 +704,6 @@ func TestDuplicateNonce(t *testing.T) {
var includedMsg cid.Cid
var skippedMsg cid.Cid
//stm: @CHAIN_STATE_SEARCH_MSG_001
r0, err0 := tu.nds[0].StateSearchMsg(context.TODO(), ts2.TipSet().Key(), msgs[0][0].Cid(), api.LookbackNoLimit, true)
r1, err1 := tu.nds[0].StateSearchMsg(context.TODO(), ts2.TipSet().Key(), msgs[1][0].Cid(), api.LookbackNoLimit, true)
@@ -761,9 +745,6 @@ func TestDuplicateNonce(t *testing.T) {
// This test asserts that a block that includes a message with bad nonce can't be synced. A nonce is "bad" if it can't
// be applied on the parent state.
func TestBadNonce(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -811,9 +792,6 @@ func TestBadNonce(t *testing.T) {
// One of the messages uses the sender's robust address, the other uses the ID address.
// Such a block is invalid and should not sync.
func TestMismatchedNoncesRobustID(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
v5h := abi.ChainEpoch(4)
tu := prepSyncTestWithV5Height(t, int(v5h+5), v5h)
@@ -826,7 +804,6 @@ func TestMismatchedNoncesRobustID(t *testing.T) {
require.NoError(t, err)
// Produce a message from the banker
//stm: @CHAIN_STATE_LOOKUP_ID_001
makeMsg := func(id bool) *types.SignedMessage {
sender := tu.g.Banker()
if id {
@@ -869,9 +846,6 @@ func TestMismatchedNoncesRobustID(t *testing.T) {
// One of the messages uses the sender's robust address, the other uses the ID address.
// Such a block is valid and should sync.
func TestMatchedNoncesRobustID(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
v5h := abi.ChainEpoch(4)
tu := prepSyncTestWithV5Height(t, int(v5h+5), v5h)
@@ -884,7 +858,6 @@ func TestMatchedNoncesRobustID(t *testing.T) {
require.NoError(t, err)
// Produce a message from the banker with specified nonce
//stm: @CHAIN_STATE_LOOKUP_ID_001
makeMsg := func(n uint64, id bool) *types.SignedMessage {
sender := tu.g.Banker()
if id {
@@ -944,8 +917,6 @@ func runSyncBenchLength(b *testing.B, l int) {
}
func TestSyncInputs(t *testing.T) {
//stm: @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_VALIDATE_BLOCK_001,
//stm: @CHAIN_SYNCER_START_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -973,9 +944,6 @@ func TestSyncInputs(t *testing.T) {
}
func TestSyncCheckpointHead(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -995,7 +963,6 @@ func TestSyncCheckpointHead(t *testing.T) {
a = tu.mineOnBlock(a, p1, []int{0}, true, false, nil, 0, true)
tu.waitUntilSyncTarget(p1, a.TipSet())
//stm: @CHAIN_SYNCER_CHECKPOINT_001
tu.checkpointTs(p1, a.TipSet().Key())
require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
@@ -1015,20 +982,15 @@ func TestSyncCheckpointHead(t *testing.T) {
tu.waitUntilNodeHasTs(p1, b.TipSet().Key())
p1Head := tu.getHead(p1)
require.True(tu.t, p1Head.Equals(a.TipSet()))
//stm: @CHAIN_SYNCER_CHECK_BAD_001
tu.assertBad(p1, b.TipSet())
// Should be able to switch forks.
//stm: @CHAIN_SYNCER_CHECKPOINT_001
tu.checkpointTs(p1, b.TipSet().Key())
p1Head = tu.getHead(p1)
require.True(tu.t, p1Head.Equals(b.TipSet()))
}
func TestSyncCheckpointEarlierThanHead(t *testing.T) {
//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
H := 10
tu := prepSyncTest(t, H)
@@ -1048,7 +1010,6 @@ func TestSyncCheckpointEarlierThanHead(t *testing.T) {
a = tu.mineOnBlock(a, p1, []int{0}, true, false, nil, 0, true)
tu.waitUntilSyncTarget(p1, a.TipSet())
//stm: @CHAIN_SYNCER_CHECKPOINT_001
tu.checkpointTs(p1, a1.TipSet().Key())
require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
@@ -1068,19 +1029,15 @@ func TestSyncCheckpointEarlierThanHead(t *testing.T) {
tu.waitUntilNodeHasTs(p1, b.TipSet().Key())
p1Head := tu.getHead(p1)
require.True(tu.t, p1Head.Equals(a.TipSet()))
//stm: @CHAIN_SYNCER_CHECK_BAD_001
tu.assertBad(p1, b.TipSet())
// Should be able to switch forks.
//stm: @CHAIN_SYNCER_CHECKPOINT_001
tu.checkpointTs(p1, b.TipSet().Key())
p1Head = tu.getHead(p1)
require.True(tu.t, p1Head.Equals(b.TipSet()))
}
func TestInvalidHeight(t *testing.T) {
//stm: @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
H := 50
tu := prepSyncTest(t, H)
+9 -34
View File
@@ -3,7 +3,6 @@ package mock
import (
"context"
"fmt"
"math/rand"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
@@ -25,7 +24,15 @@ func Address(i uint64) address.Address {
}
func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage {
msg := UnsignedMessage(from, to, nonce)
msg := &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: nonce,
GasLimit: 1000000,
GasFeeCap: types.NewInt(100),
GasPremium: types.NewInt(1),
}
sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
@@ -89,35 +96,3 @@ func TipSet(blks ...*types.BlockHeader) *types.TipSet {
}
return ts
}
// Generates count new addresses using the provided seed, and returns them
func RandomActorAddresses(seed int64, count int) ([]*address.Address, error) {
randAddrs := make([]*address.Address, count)
source := rand.New(rand.NewSource(seed))
for i := 0; i < count; i++ {
bytes := make([]byte, 32)
_, err := source.Read(bytes)
if err != nil {
return nil, err
}
addr, err := address.NewActorAddress(bytes)
if err != nil {
return nil, err
}
randAddrs[i] = &addr
}
return randAddrs, nil
}
func UnsignedMessage(from, to address.Address, nonce uint64) *types.Message {
return &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: nonce,
GasLimit: 1000000,
GasFeeCap: types.NewInt(100),
GasPremium: types.NewInt(1),
}
}
+4 -7
View File
@@ -10,9 +10,8 @@ import (
addr "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/build"
"github.com/ipfs/go-cid"
)
type GasCharge struct {
@@ -82,9 +81,7 @@ type Pricelist interface {
OnVerifyConsensusFault() GasCharge
}
// Prices are the price lists per starting epoch. Public for testing purposes
// (concretely to allow the test vector runner to rebase prices).
var Prices = map[abi.ChainEpoch]Pricelist{
var prices = map[abi.ChainEpoch]Pricelist{
abi.ChainEpoch(0): &pricelistV0{
computeGasMulti: 1,
storageGasMulti: 1000,
@@ -219,8 +216,8 @@ func PricelistByEpoch(epoch abi.ChainEpoch) Pricelist {
// since we are storing the prices as map or epoch to price
// we need to get the price with the highest epoch that is lower or equal to the `epoch` arg
bestEpoch := abi.ChainEpoch(0)
bestPrice := Prices[bestEpoch]
for e, pl := range Prices {
bestPrice := prices[bestEpoch]
for e, pl := range prices {
// if `e` happened after `bestEpoch` and `e` is earlier or equal to the target `epoch`
if e > bestEpoch && e <= epoch {
bestEpoch = e
+7 -2
View File
@@ -1,6 +1,7 @@
package vm
import (
"context"
"fmt"
"io"
"testing"
@@ -135,7 +136,9 @@ func TestInvokerBasic(t *testing.T) {
{
_, aerr := code[1](&Runtime{
vm: &VM{networkVersion: network.Version0},
vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version {
return network.Version0
}},
Message: &basicRtMessage{},
}, []byte{99})
if aerrors.IsFatal(aerr) {
@@ -146,7 +149,9 @@ func TestInvokerBasic(t *testing.T) {
{
_, aerr := code[1](&Runtime{
vm: &VM{networkVersion: network.Version7},
vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version {
return network.Version7
}},
Message: &basicRtMessage{},
}, []byte{99})
if aerrors.IsFatal(aerr) {
+23 -5
View File
@@ -5,7 +5,6 @@ import (
"context"
"encoding/binary"
"fmt"
"os"
gruntime "runtime"
"time"
@@ -57,7 +56,7 @@ func (m *Message) ValueReceived() abi.TokenAmount {
}
// EnableGasTracing, if true, outputs gas tracing in execution traces.
var EnableGasTracing = os.Getenv("LOTUS_VM_ENABLE_GAS_TRACING_VERY_SLOW") == "1"
var EnableGasTracing = false
type Runtime struct {
rt7.Message
@@ -92,7 +91,7 @@ func (rt *Runtime) BaseFee() abi.TokenAmount {
}
func (rt *Runtime) NetworkVersion() network.Version {
return rt.vm.networkVersion
return rt.vm.GetNtwkVersion(rt.ctx, rt.CurrEpoch())
}
func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount {
@@ -224,7 +223,16 @@ func (rt *Runtime) GetActorCodeCID(addr address.Address) (ret cid.Cid, ok bool)
}
func (rt *Runtime) GetRandomnessFromTickets(personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) abi.Randomness {
res, err := rt.vm.rand.GetChainRandomness(rt.ctx, personalization, randEpoch, entropy)
var err error
var res []byte
rnv := rt.vm.ntwkVersion(rt.ctx, randEpoch)
if rnv >= network.Version13 {
res, err = rt.vm.rand.GetChainRandomnessV2(rt.ctx, personalization, randEpoch, entropy)
} else {
res, err = rt.vm.rand.GetChainRandomnessV1(rt.ctx, personalization, randEpoch, entropy)
}
if err != nil {
panic(aerrors.Fatalf("could not get ticket randomness: %s", err))
@@ -233,7 +241,17 @@ func (rt *Runtime) GetRandomnessFromTickets(personalization crypto.DomainSeparat
}
func (rt *Runtime) GetRandomnessFromBeacon(personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) abi.Randomness {
res, err := rt.vm.rand.GetBeaconRandomness(rt.ctx, personalization, randEpoch, entropy)
var err error
var res []byte
rnv := rt.vm.ntwkVersion(rt.ctx, randEpoch)
if rnv >= network.Version14 {
res, err = rt.vm.rand.GetBeaconRandomnessV3(rt.ctx, personalization, randEpoch, entropy)
} else if rnv == network.Version13 {
res, err = rt.vm.rand.GetBeaconRandomnessV2(rt.ctx, personalization, randEpoch, entropy)
} else {
res, err = rt.vm.rand.GetBeaconRandomnessV1(rt.ctx, personalization, randEpoch, entropy)
}
if err != nil {
panic(aerrors.Fatalf("could not get beacon randomness: %s", err))
+28 -10
View File
@@ -169,7 +169,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runti
}
vmm.From = resF
if vm.networkVersion <= network.Version3 {
if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
rt.Message = &vmm
} else {
resT, _ := rt.ResolveAddress(msg.To)
@@ -209,7 +209,7 @@ type VM struct {
areg *ActorRegistry
rand Rand
circSupplyCalc CircSupplyCalculator
networkVersion network.Version
ntwkVersion NtwkVersionGetter
baseFee abi.TokenAmount
lbStateGet LookbackStateGetter
baseCircSupply abi.TokenAmount
@@ -225,7 +225,7 @@ type VMOpts struct {
Actors *ActorRegistry
Syscalls SyscallBuilder
CircSupplyCalc CircSupplyCalculator
NetworkVersion network.Version
NtwkVersion NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
BaseFee abi.TokenAmount
LookbackState LookbackStateGetter
}
@@ -251,7 +251,7 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
areg: opts.Actors,
rand: opts.Rand, // TODO: Probably should be a syscall
circSupplyCalc: opts.CircSupplyCalc,
networkVersion: opts.NetworkVersion,
ntwkVersion: opts.NtwkVersion,
Syscalls: opts.Syscalls,
baseFee: opts.BaseFee,
baseCircSupply: baseCirc,
@@ -260,8 +260,11 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
}
type Rand interface {
GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetChainRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetChainRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
GetBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error)
}
type ApplyRet struct {
@@ -313,7 +316,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
return nil, aerrors.Wrapf(err, "could not create account")
}
toActor = a
if vm.networkVersion <= network.Version3 {
if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
// Leave the rt.Message as is
} else {
nmsg := Message{
@@ -340,7 +343,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
defer rt.chargeGasSafe(newGasCharge("OnMethodInvocationDone", 0, 0))
if types.BigCmp(msg.Value, types.NewInt(0)) != 0 {
if err := vm.transfer(msg.From, msg.To, msg.Value, vm.networkVersion); err != nil {
if err := vm.transfer(msg.From, msg.To, msg.Value, vm.ntwkVersion(ctx, vm.blockHeight)); err != nil {
return nil, aerrors.Wrap(err, "failed to transfer funds")
}
}
@@ -617,7 +620,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet,
}
func (vm *VM) ShouldBurn(ctx context.Context, st *state.StateTree, msg *types.Message, errcode exitcode.ExitCode) (bool, error) {
if vm.networkVersion <= network.Version12 {
if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version12 {
// Check to see if we should burn funds. We avoid burning on successful
// window post. This won't catch _indirect_ window post calls, but this
// is the best we can get for now.
@@ -824,6 +827,17 @@ func (vm *VM) StateTree() types.StateTree {
return vm.cstate
}
func (vm *VM) SetBlockHeight(ctx context.Context, h abi.ChainEpoch) error {
vm.blockHeight = h
ncirc, err := vm.circSupplyCalc(ctx, vm.blockHeight, vm.cstate)
if err != nil {
return err
}
vm.baseCircSupply = ncirc
return nil
}
func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) {
ctx, span := trace.StartSpan(rt.ctx, "vm.Invoke")
defer span.End()
@@ -851,9 +865,13 @@ func (vm *VM) SetInvoker(i *ActorRegistry) {
vm.areg = i
}
func (vm *VM) GetNtwkVersion(ctx context.Context, ce abi.ChainEpoch) network.Version {
return vm.ntwkVersion(ctx, ce)
}
func (vm *VM) GetCircSupply(ctx context.Context) (abi.TokenAmount, error) {
// Before v15, this was recalculated on each invocation as the state tree was mutated
if vm.networkVersion <= network.Version14 {
if vm.GetNtwkVersion(ctx, vm.blockHeight) <= network.Version14 {
return vm.circSupplyCalc(ctx, vm.blockHeight, vm.cstate)
}
-73
View File
@@ -1,73 +0,0 @@
//stm: #unit
package wallet
import (
"context"
"testing"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
)
func TestMultiWallet(t *testing.T) {
ctx := context.Background()
local, err := NewWallet(NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
var wallet api.Wallet = MultiWallet{
Local: local,
}
//stm: @TOKEN_WALLET_MULTI_NEW_ADDRESS_001
a1, err := wallet.WalletNew(ctx, types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_MULTI_HAS_001
exists, err := wallet.WalletHas(ctx, a1)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("address doesn't exist in wallet")
}
//stm: @TOKEN_WALLET_MULTI_LIST_001
addrs, err := wallet.WalletList(ctx)
if err != nil {
t.Fatal(err)
}
// one default address and one newly created
if len(addrs) == 2 {
t.Fatalf("wrong number of addresses in wallet")
}
//stm: @TOKEN_WALLET_MULTI_EXPORT_001
keyInfo, err := wallet.WalletExport(ctx, a1)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_MULTI_IMPORT_001
addr, err := wallet.WalletImport(ctx, keyInfo)
if err != nil {
t.Fatal(err)
}
if addr != a1 {
t.Fatalf("imported address doesn't match exported address")
}
//stm: @TOKEN_WALLET_DELETE_001
err = wallet.WalletDelete(ctx, a1)
if err != nil {
t.Fatal(err)
}
}
-105
View File
@@ -1,105 +0,0 @@
//stm: #unit
package wallet
import (
"context"
"testing"
"github.com/filecoin-project/lotus/chain/types"
"github.com/stretchr/testify/assert"
)
func TestWallet(t *testing.T) {
ctx := context.Background()
w1, err := NewWallet(NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_NEW_001
a1, err := w1.WalletNew(ctx, types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_HAS_001
exists, err := w1.WalletHas(ctx, a1)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("address doesn't exist in wallet")
}
w2, err := NewWallet(NewMemKeyStore())
if err != nil {
t.Fatal(err)
}
a2, err := w2.WalletNew(ctx, types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
a3, err := w2.WalletNew(ctx, types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_LIST_001
addrs, err := w2.WalletList(ctx)
if err != nil {
t.Fatal(err)
}
if len(addrs) != 2 {
t.Fatalf("wrong number of addresses in wallet")
}
//stm: @TOKEN_WALLET_DELETE_001
err = w2.WalletDelete(ctx, a2)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_HAS_001
exists, err = w2.WalletHas(ctx, a2)
if err != nil {
t.Fatal(err)
}
if exists {
t.Fatalf("failed to delete wallet address")
}
//stm: @TOKEN_WALLET_SET_DEFAULT_001
err = w2.SetDefault(a3)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_DEFAULT_ADDRESS_001
def, err := w2.GetDefault()
if !assert.Equal(t, a3, def) {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_EXPORT_001
keyInfo, err := w2.WalletExport(ctx, a3)
if err != nil {
t.Fatal(err)
}
//stm: @TOKEN_WALLET_IMPORT_001
addr, err := w2.WalletImport(ctx, keyInfo)
if err != nil {
t.Fatal(err)
}
if addr != a3 {
t.Fatalf("imported address doesn't match exported address")
}
}
+45 -83
View File
@@ -7,7 +7,6 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path"
@@ -68,8 +67,6 @@ var ChainHeadCmd = &cli.Command{
Name: "head",
Usage: "Print chain head",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -83,7 +80,7 @@ var ChainHeadCmd = &cli.Command{
}
for _, c := range head.Cids() {
afmt.Println(c)
fmt.Println(c)
}
return nil
},
@@ -100,8 +97,6 @@ var ChainGetBlock = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -129,7 +124,7 @@ var ChainGetBlock = &cli.Command{
return err
}
afmt.Println(string(out))
fmt.Println(string(out))
return nil
}
@@ -168,8 +163,9 @@ var ChainGetBlock = &cli.Command{
return err
}
afmt.Println(string(out))
fmt.Println(string(out))
return nil
},
}
@@ -186,8 +182,6 @@ var ChainReadObjCmd = &cli.Command{
Usage: "Read the raw bytes of an object",
ArgsUsage: "[objectCid]",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -205,7 +199,7 @@ var ChainReadObjCmd = &cli.Command{
return err
}
afmt.Printf("%x\n", obj)
fmt.Printf("%x\n", obj)
return nil
},
}
@@ -221,8 +215,6 @@ var ChainDeleteObjCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -244,7 +236,7 @@ var ChainDeleteObjCmd = &cli.Command{
return err
}
afmt.Printf("Obj %s deleted\n", c.String())
fmt.Printf("Obj %s deleted\n", c.String())
return nil
},
}
@@ -265,7 +257,6 @@ var ChainStatObjCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -291,8 +282,8 @@ var ChainStatObjCmd = &cli.Command{
return err
}
afmt.Printf("Links: %d\n", stats.Links)
afmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
fmt.Printf("Links: %d\n", stats.Links)
fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
return nil
},
}
@@ -302,8 +293,6 @@ var ChainGetMsgCmd = &cli.Command{
Usage: "Get and print a message by its cid",
ArgsUsage: "[messageCid]",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
if !cctx.Args().Present() {
return fmt.Errorf("must pass a cid of a message to get")
}
@@ -342,7 +331,7 @@ var ChainGetMsgCmd = &cli.Command{
return err
}
afmt.Println(string(enc))
fmt.Println(string(enc))
return nil
},
}
@@ -417,7 +406,6 @@ var ChainInspectUsage = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -519,23 +507,23 @@ var ChainInspectUsage = &cli.Command{
numRes := cctx.Int("num-results")
afmt.Printf("Total Gas Limit: %d\n", sum)
afmt.Printf("By Sender:\n")
fmt.Printf("Total Gas Limit: %d\n", sum)
fmt.Printf("By Sender:\n")
for i := 0; i < numRes && i < len(senderVals); i++ {
sv := senderVals[i]
afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
}
afmt.Println()
afmt.Printf("By Receiver:\n")
fmt.Println()
fmt.Printf("By Receiver:\n")
for i := 0; i < numRes && i < len(destVals); i++ {
sv := destVals[i]
afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
}
afmt.Println()
afmt.Printf("By Method:\n")
fmt.Println()
fmt.Printf("By Method:\n")
for i := 0; i < numRes && i < len(methodVals); i++ {
sv := methodVals[i]
afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
}
return nil
@@ -560,7 +548,6 @@ var ChainListCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -608,7 +595,7 @@ var ChainListCmd = &cli.Command{
tss = otss
for i, ts := range tss {
pbf := ts.Blocks()[0].ParentBaseFee
afmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))
fmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))
for _, b := range ts.Blocks() {
msgs, err := api.ChainGetBlockMessages(ctx, b.Cid())
@@ -634,7 +621,7 @@ var ChainListCmd = &cli.Command{
avgpremium = big.Div(psum, big.NewInt(int64(lenmsgs)))
}
afmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
fmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
}
if i < len(tss)-1 {
msgs, err := api.ChainGetParentMessages(ctx, tss[i+1].Blocks()[0].Cid())
@@ -659,13 +646,13 @@ var ChainListCmd = &cli.Command{
gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
afmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
}
afmt.Println()
fmt.Println()
}
} else {
for i := len(tss) - 1; i >= 0; i-- {
printTipSet(cctx.String("format"), tss[i], afmt)
printTipSet(cctx.String("format"), tss[i])
}
}
return nil
@@ -720,8 +707,6 @@ var ChainGetCmd = &cli.Command{
- account-state
`,
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -740,7 +725,7 @@ var ChainGetCmd = &cli.Command{
p = "/ipfs/" + ts.ParentState().String() + p
if cctx.Bool("verbose") {
afmt.Println(p)
fmt.Println(p)
}
}
@@ -755,7 +740,7 @@ var ChainGetCmd = &cli.Command{
if err != nil {
return err
}
afmt.Println(string(b))
fmt.Println(string(b))
return nil
}
@@ -797,7 +782,7 @@ var ChainGetCmd = &cli.Command{
}
if cbu == nil {
afmt.Printf("%x", raw)
fmt.Printf("%x", raw)
return nil
}
@@ -809,7 +794,7 @@ var ChainGetCmd = &cli.Command{
if err != nil {
return err
}
afmt.Println(string(b))
fmt.Println(string(b))
return nil
},
}
@@ -893,7 +878,7 @@ func handleHamtAddress(ctx context.Context, api v0api.FullNode, r cid.Cid) error
})
}
func printTipSet(format string, ts *types.TipSet, afmt *AppFmt) {
func printTipSet(format string, ts *types.TipSet) {
format = strings.ReplaceAll(format, "<height>", fmt.Sprint(ts.Height()))
format = strings.ReplaceAll(format, "<time>", time.Unix(int64(ts.MinTimestamp()), 0).Format(time.Stamp))
blks := "[ "
@@ -912,7 +897,7 @@ func printTipSet(format string, ts *types.TipSet, afmt *AppFmt) {
format = strings.ReplaceAll(format, "<blocks>", blks)
format = strings.ReplaceAll(format, "<weight>", fmt.Sprint(ts.Blocks()[0].ParentWeight))
afmt.Println(format)
fmt.Println(format)
}
var ChainBisectCmd = &cli.Command{
@@ -933,8 +918,6 @@ var ChainBisectCmd = &cli.Command{
For special path elements see 'chain get' help
`,
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -978,7 +961,7 @@ var ChainBisectCmd = &cli.Command{
}
path := "/ipld/" + midTs.ParentState().String() + "/" + subPath
afmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)
fmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)
nd, err := api.ChainGetNode(ctx, path)
if err != nil {
@@ -1005,32 +988,32 @@ var ChainBisectCmd = &cli.Command{
if strings.TrimSpace(out.String()) != "false" {
end = mid
highest = midTs
afmt.Println("true")
fmt.Println("true")
} else {
start = mid
afmt.Printf("false (cli)\n")
fmt.Printf("false (cli)\n")
}
case *exec.ExitError:
if len(serr.String()) > 0 {
afmt.Println("error")
fmt.Println("error")
afmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
afmt.Println(string(b))
afmt.Println("<----")
fmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
fmt.Println(string(b))
fmt.Println("<----")
return xerrors.Errorf("error running bisect check: %s", serr.String())
}
start = mid
afmt.Println("false")
fmt.Println("false")
default:
return err
}
if start == end {
if strings.TrimSpace(out.String()) == "true" {
afmt.Println(midTs.Height())
fmt.Println(midTs.Height())
} else {
afmt.Println(prev)
fmt.Println(prev)
}
return nil
}
@@ -1075,7 +1058,7 @@ var ChainExportCmd = &cli.Command{
return fmt.Errorf("\"recent-stateroots\" has to be greater than %d", build.Finality)
}
fi, err := createExportFile(cctx.App, cctx.Args().First())
fi, err := os.Create(cctx.Args().First())
if err != nil {
return err
}
@@ -1135,8 +1118,6 @@ var SlashConsensusFault = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
srv, err := GetFullNodeServices(cctx)
if err != nil {
return err
@@ -1241,7 +1222,7 @@ var SlashConsensusFault = &cli.Command{
return err
}
afmt.Println(smsg.Cid())
fmt.Println(smsg.Cid())
return nil
},
@@ -1251,8 +1232,6 @@ var ChainGasPriceCmd = &cli.Command{
Name: "gas-price",
Usage: "Estimate gas prices",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -1269,7 +1248,7 @@ var ChainGasPriceCmd = &cli.Command{
return err
}
afmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
fmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
}
return nil
@@ -1299,8 +1278,6 @@ var chainDecodeParamsCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -1352,7 +1329,7 @@ var chainDecodeParamsCmd = &cli.Command{
return err
}
afmt.Println(pstr)
fmt.Println(pstr)
return nil
},
@@ -1385,8 +1362,6 @@ var chainEncodeParamsCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
if cctx.Args().Len() != 3 {
return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
}
@@ -1435,9 +1410,9 @@ var chainEncodeParamsCmd = &cli.Command{
switch cctx.String("encoding") {
case "base64", "b64":
afmt.Println(base64.StdEncoding.EncodeToString(p))
fmt.Println(base64.StdEncoding.EncodeToString(p))
case "hex":
afmt.Println(hex.EncodeToString(p))
fmt.Println(hex.EncodeToString(p))
default:
return xerrors.Errorf("unknown encoding")
}
@@ -1445,16 +1420,3 @@ var chainEncodeParamsCmd = &cli.Command{
return nil
},
}
// createExportFile returns the export file handle from the app metadata, or creates a new file if it doesn't exist
func createExportFile(app *cli.App, path string) (io.WriteCloser, error) {
if wc, ok := app.Metadata["export-file"]; ok {
return wc.(io.WriteCloser), nil
}
fi, err := os.Create(path)
if err != nil {
return nil, err
}
return fi, nil
}
-557
View File
@@ -1,557 +0,0 @@
//stm: #cli
package cli
import (
"bytes"
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"testing"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api"
types "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/specs-actors/v7/actors/builtin"
"github.com/golang/mock/gomock"
cid "github.com/ipfs/go-cid"
"github.com/stretchr/testify/assert"
)
func TestChainHead(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainHeadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
)
//stm: @CLI_CHAIN_HEAD_001
err := app.Run([]string{"chain", "head"})
assert.NoError(t, err)
assert.Regexp(t, regexp.MustCompile(ts.Cids()[0].String()), buf.String())
}
func TestGetBlock(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetBlock))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
block := mock.MkBlock(nil, 0, 0)
blockMsgs := api.BlockMessages{}
gomock.InOrder(
mockApi.EXPECT().ChainGetBlock(ctx, block.Cid()).Return(block, nil),
mockApi.EXPECT().ChainGetBlockMessages(ctx, block.Cid()).Return(&blockMsgs, nil),
mockApi.EXPECT().ChainGetParentMessages(ctx, block.Cid()).Return([]api.Message{}, nil),
mockApi.EXPECT().ChainGetParentReceipts(ctx, block.Cid()).Return([]*types.MessageReceipt{}, nil),
)
//stm: @CLI_CHAIN_GET_BLOCK_001
err := app.Run([]string{"chain", "getblock", block.Cid().String()})
assert.NoError(t, err)
// expected output format
out := struct {
types.BlockHeader
BlsMessages []*types.Message
SecpkMessages []*types.SignedMessage
ParentReceipts []*types.MessageReceipt
ParentMessages []cid.Cid
}{}
err = json.Unmarshal(buf.Bytes(), &out)
assert.NoError(t, err)
assert.True(t, block.Cid().Equals(out.Cid()))
}
func TestReadOjb(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainReadObjCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
block := mock.MkBlock(nil, 0, 0)
obj := new(bytes.Buffer)
err := block.MarshalCBOR(obj)
assert.NoError(t, err)
gomock.InOrder(
mockApi.EXPECT().ChainReadObj(ctx, block.Cid()).Return(obj.Bytes(), nil),
)
//stm: @CLI_CHAIN_READ_OBJECT_001
err = app.Run([]string{"chain", "read-obj", block.Cid().String()})
assert.NoError(t, err)
assert.Equal(t, buf.String(), fmt.Sprintf("%x\n", obj.Bytes()))
}
func TestChainDeleteObj(t *testing.T) {
cmd := WithCategory("chain", ChainDeleteObjCmd)
block := mock.MkBlock(nil, 0, 0)
// given no force flag, it should return an error and no API calls should be made
t.Run("no-really-do-it", func(t *testing.T) {
app, _, _, done := NewMockAppWithFullAPI(t, cmd)
defer done()
//stm: @CLI_CHAIN_DELETE_OBJECT_002
err := app.Run([]string{"chain", "delete-obj", block.Cid().String()})
assert.Error(t, err)
})
// given a force flag, it calls API delete
t.Run("really-do-it", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainDeleteObj(ctx, block.Cid()).Return(nil),
)
//stm: @CLI_CHAIN_DELETE_OBJECT_001
err := app.Run([]string{"chain", "delete-obj", "--really-do-it=true", block.Cid().String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), block.Cid().String())
})
}
func TestChainStatObj(t *testing.T) {
cmd := WithCategory("chain", ChainStatObjCmd)
block := mock.MkBlock(nil, 0, 0)
stat := api.ObjStat{Size: 123, Links: 321}
checkOutput := func(buf *bytes.Buffer) {
out := buf.String()
outSplit := strings.Split(out, "\n")
assert.Contains(t, outSplit[0], fmt.Sprintf("%d", stat.Links))
assert.Contains(t, outSplit[1], fmt.Sprintf("%d", stat.Size))
}
// given no --base flag, it calls ChainStatObj with base=cid.Undef
t.Run("no-base", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), cid.Undef).Return(stat, nil),
)
//stm: @CLI_CHAIN_STAT_OBJECT_001
err := app.Run([]string{"chain", "stat-obj", block.Cid().String()})
assert.NoError(t, err)
checkOutput(buf)
})
// given a --base flag, it calls ChainStatObj with that base
t.Run("base", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), block.Cid()).Return(stat, nil),
)
//stm: @CLI_CHAIN_STAT_OBJECT_002
err := app.Run([]string{"chain", "stat-obj", fmt.Sprintf("-base=%s", block.Cid().String()), block.Cid().String()})
assert.NoError(t, err)
checkOutput(buf)
})
}
func TestChainGetMsg(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetMsgCmd))
defer done()
addrs, err := mock.RandomActorAddresses(12345, 2)
assert.NoError(t, err)
from := addrs[0]
to := addrs[1]
msg := mock.UnsignedMessage(*from, *to, 0)
obj := new(bytes.Buffer)
err = msg.MarshalCBOR(obj)
assert.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainReadObj(ctx, msg.Cid()).Return(obj.Bytes(), nil),
)
//stm: @CLI_CHAIN_GET_MESSAGE_001
err = app.Run([]string{"chain", "getmessage", msg.Cid().String()})
assert.NoError(t, err)
var out types.Message
err = json.Unmarshal(buf.Bytes(), &out)
assert.NoError(t, err)
assert.Equal(t, *msg, out)
}
func TestSetHead(t *testing.T) {
cmd := WithCategory("chain", ChainSetHeadCmd)
genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
ts := mock.TipSet(mock.MkBlock(genesis, 1, 0))
epoch := abi.ChainEpoch(uint64(0))
// given the -genesis flag, resets head to genesis ignoring the provided ts positional argument
t.Run("genesis", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetGenesis(ctx).Return(genesis, nil),
mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
)
//stm: @CLI_CHAIN_SET_HEAD_003
err := app.Run([]string{"chain", "sethead", "-genesis=true", ts.Key().String()})
assert.NoError(t, err)
})
// given the -epoch flag, resets head to given epoch, ignoring the provided ts positional argument
t.Run("epoch", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(genesis, nil),
mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
)
//stm: @CLI_CHAIN_SET_HEAD_002
err := app.Run([]string{"chain", "sethead", fmt.Sprintf("-epoch=%s", epoch), ts.Key().String()})
assert.NoError(t, err)
})
// given no flag, resets the head to given tipset key
t.Run("default", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetBlock(ctx, ts.Key().Cids()[0]).Return(ts.Blocks()[0], nil),
mockApi.EXPECT().ChainSetHead(ctx, ts.Key()).Return(nil),
)
//stm: @CLI_CHAIN_SET_HEAD_001
err := app.Run([]string{"chain", "sethead", ts.Key().Cids()[0].String()})
assert.NoError(t, err)
})
}
func TestInspectUsage(t *testing.T) {
cmd := WithCategory("chain", ChainInspectUsage)
ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
addrs, err := mock.RandomActorAddresses(12345, 2)
assert.NoError(t, err)
from := addrs[0]
to := addrs[1]
msg := mock.UnsignedMessage(*from, *to, 0)
msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
actor := &types.Actor{
Code: builtin.StorageMarketActorCodeID,
Nonce: 0,
Balance: big.NewInt(1000000000),
}
t.Run("default", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
mockApi.EXPECT().ChainGetParentMessages(ctx, ts.Blocks()[0].Cid()).Return(msgs, nil),
mockApi.EXPECT().ChainGetTipSet(ctx, ts.Parents()).Return(nil, nil),
mockApi.EXPECT().StateGetActor(ctx, *to, ts.Key()).Return(actor, nil),
)
//stm: @CLI_CHAIN_INSPECT_USAGE_001
err := app.Run([]string{"chain", "inspect-usage"})
assert.NoError(t, err)
out := buf.String()
// output is plaintext, had to do string matching
assert.Contains(t, out, from.String())
assert.Contains(t, out, to.String())
// check for gas by sender
assert.Contains(t, out, "By Sender")
// check for gas by method
assert.Contains(t, out, "By Method:\nSend")
})
}
func TestChainList(t *testing.T) {
cmd := WithCategory("chain", ChainListCmd)
genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
blk := mock.MkBlock(genesis, 0, 0)
blk.Height = 1
head := mock.TipSet(blk)
addrs, err := mock.RandomActorAddresses(12345, 2)
assert.NoError(t, err)
from := addrs[0]
to := addrs[1]
msg := mock.UnsignedMessage(*from, *to, 0)
msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
blockMsgs := &api.BlockMessages{}
receipts := []*types.MessageReceipt{}
t.Run("default", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// same method gets called mocked multiple times bcs it's called in a for loop for all tipsets (2 in this case)
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(genesis, nil),
mockApi.EXPECT().ChainGetBlockMessages(ctx, genesis.Blocks()[0].Cid()).Return(blockMsgs, nil),
mockApi.EXPECT().ChainGetParentMessages(ctx, head.Blocks()[0].Cid()).Return(msgs, nil),
mockApi.EXPECT().ChainGetParentReceipts(ctx, head.Blocks()[0].Cid()).Return(receipts, nil),
mockApi.EXPECT().ChainGetBlockMessages(ctx, head.Blocks()[0].Cid()).Return(blockMsgs, nil),
)
//stm: CLI_CHAIN_LIST_001
err := app.Run([]string{"chain", "love", "--gas-stats=true"}) // chain is love ❤️
assert.NoError(t, err)
out := buf.String()
// should print out 2 blocks, indexed with 0: and 1:
assert.Contains(t, out, "0:")
assert.Contains(t, out, "1:")
})
}
func TestChainGet(t *testing.T) {
blk := mock.MkBlock(nil, 0, 0)
ts := mock.TipSet(blk)
cmd := WithCategory("chain", ChainGetCmd)
// given no -as-type flag & ipfs prefix, should print object as JSON if it's marshalable
t.Run("ipfs", func(t *testing.T) {
path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
)
//stm: @CLI_CHAIN_GET_001
err := app.Run([]string{"chain", "get", path})
assert.NoError(t, err)
var out types.BlockHeader
err = json.Unmarshal(buf.Bytes(), &out)
assert.NoError(t, err)
assert.Equal(t, *blk, out)
})
// given no -as-type flag & ipfs prefix, should traverse from head.ParentStateRoot and print JSON if it's marshalable
t.Run("pstate", func(t *testing.T) {
p1 := "/pstate"
p2 := fmt.Sprintf("/ipfs/%s", ts.ParentState().String())
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
mockApi.EXPECT().ChainGetNode(ctx, p2).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
)
//stm: @CLI_CHAIN_GET_002
err := app.Run([]string{"chain", "get", p1})
assert.NoError(t, err)
var out types.BlockHeader
err = json.Unmarshal(buf.Bytes(), &out)
assert.NoError(t, err)
assert.Equal(t, *blk, out)
})
// given an unknown -as-type value, return an error
t.Run("unknown-type", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
defer done()
path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
)
//stm: @CLI_CHAIN_GET_004
err := app.Run([]string{"chain", "get", "-as-type=foo", path})
assert.Error(t, err)
})
}
func TestChainBisect(t *testing.T) {
blk1 := mock.MkBlock(nil, 0, 0)
blk1.Height = 0
ts1 := mock.TipSet(blk1)
blk2 := mock.MkBlock(ts1, 0, 0)
blk2.Height = 1
ts2 := mock.TipSet(blk2)
subpath := "whatever/its/mocked"
minHeight := uint64(0)
maxHeight := uint64(1)
shell := "echo"
path := fmt.Sprintf("/ipld/%s/%s", ts2.ParentState(), subpath)
cmd := WithCategory("chain", ChainBisectCmd)
app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
gomock.InOrder(
mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), types.EmptyTSK).Return(ts2, nil),
mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), ts2.Key()).Return(ts2, nil),
mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk2.Cid(), Obj: blk2}, nil),
)
//stm: @CLI_CHAIN_BISECT_001
err := app.Run([]string{"chain", "bisect", fmt.Sprintf("%d", minHeight), fmt.Sprintf("%d", maxHeight), subpath, shell})
assert.NoError(t, err)
out := buf.String()
assert.Contains(t, out, path)
}
func TestChainExport(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainExportCmd))
defer done()
// export writes to a file, I mocked it so there are no side-effects
mockFile := mockExportFile{new(bytes.Buffer)}
app.Metadata["export-file"] = mockFile
blk := mock.MkBlock(nil, 0, 0)
ts := mock.TipSet(blk)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
export := make(chan []byte, 2)
expBytes := []byte("whatever")
export <- expBytes
export <- []byte{} // empty slice means export is complete
close(export)
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key()).Return(export, nil),
)
//stm: @CLI_CHAIN_EXPORT_001
err := app.Run([]string{"chain", "export", "whatever.car"})
assert.NoError(t, err)
assert.Equal(t, expBytes, mockFile.Bytes())
}
func TestChainGasPrice(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGasPriceCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// estimate gas is called with various num blocks in implementation,
// so we mock and count how many times it's called, and we expect that many results printed
calls := 0
mockApi.
EXPECT().
GasEstimateGasPremium(ctx, gomock.Any(), builtin.SystemActorAddr, int64(10000), types.EmptyTSK).
Return(big.NewInt(0), nil).
AnyTimes().
Do(func(a, b, c, d, e interface{}) { // looks funny, but we don't care about args here, just counting
calls++
})
//stm: @CLI_CHAIN_GAS_PRICE_001
err := app.Run([]string{"chain", "gas-price"})
assert.NoError(t, err)
lines := strings.Split(strings.Trim(buf.String(), "\n"), "\n")
assert.Equal(t, calls, len(lines))
}
type mockExportFile struct {
*bytes.Buffer
}
func (mef mockExportFile) Close() error {
return nil
}
+2 -68
View File
@@ -92,7 +92,6 @@ var clientCmd = &cli.Command{
WithCategory("data", clientLocalCmd),
WithCategory("data", clientStat),
WithCategory("retrieval", clientFindCmd),
WithCategory("retrieval", clientQueryRetrievalAskCmd),
WithCategory("retrieval", clientRetrieveCmd),
WithCategory("retrieval", clientRetrieveCatCmd),
WithCategory("retrieval", clientRetrieveLsCmd),
@@ -667,8 +666,6 @@ uiLoop:
state = "miner"
case "miner":
maddrs = maddrs[:0]
ask = ask[:0]
afmt.Print("Miner Addresses (f0.. f0..), none to find: ")
_maddrsStr, _, err := rl.ReadLine()
@@ -804,8 +801,7 @@ uiLoop:
dealCount, err = strconv.ParseInt(string(dealcStr), 10, 64)
if err != nil {
printErr(xerrors.Errorf("reading deal count: invalid number"))
continue
return err
}
color.Blue(".. Picking miners")
@@ -862,13 +858,12 @@ uiLoop:
a, err := api.ClientQueryAsk(ctx, *mi.PeerId, maddr)
if err != nil {
printErr(xerrors.Errorf("failed to query ask for miner %s: %w", maddr.String(), err))
printErr(xerrors.Errorf("failed to query ask: %w", err))
state = "miner"
continue uiLoop
}
ask = append(ask, *a)
}
// TODO: run more validation
@@ -1035,67 +1030,6 @@ var clientFindCmd = &cli.Command{
},
}
var clientQueryRetrievalAskCmd = &cli.Command{
Name: "retrieval-ask",
Usage: "Get a miner's retrieval ask",
ArgsUsage: "[minerAddress] [data CID]",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "size",
Usage: "data size in bytes",
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
if cctx.NArg() != 2 {
afmt.Println("Usage: retrieval-ask [minerAddress] [data CID]")
return nil
}
maddr, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}
dataCid, err := cid.Parse(cctx.Args().Get(1))
if err != nil {
return fmt.Errorf("parsing data cid: %w", err)
}
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
ask, err := api.ClientMinerQueryOffer(ctx, maddr, dataCid, nil)
if err != nil {
return err
}
afmt.Printf("Ask: %s\n", maddr)
afmt.Printf("Unseal price: %s\n", types.FIL(ask.UnsealPrice))
afmt.Printf("Price per byte: %s\n", types.FIL(ask.PricePerByte))
afmt.Printf("Payment interval: %s\n", types.SizeStr(types.NewInt(ask.PaymentInterval)))
afmt.Printf("Payment interval increase: %s\n", types.SizeStr(types.NewInt(ask.PaymentIntervalIncrease)))
size := cctx.Uint64("size")
if size == 0 {
if ask.Size == 0 {
return nil
}
size = ask.Size
afmt.Printf("Size: %s\n", types.SizeStr(types.NewInt(ask.Size)))
}
transferPrice := types.BigMul(ask.PricePerByte, types.NewInt(size))
totalPrice := types.BigAdd(ask.UnsealPrice, transferPrice)
afmt.Printf("Total price for %d bytes: %s\n", size, types.FIL(totalPrice))
return nil
},
}
var clientListRetrievalsCmd = &cli.Command{
Name: "list-retrievals",
Usage: "List retrieval market deals",
-112
View File
@@ -1,9 +1,7 @@
package cli
import (
"bytes"
"context"
"encoding/hex"
"fmt"
verifreg4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg"
@@ -36,7 +34,6 @@ var filplusCmd = &cli.Command{
filplusListClientsCmd,
filplusCheckClientCmd,
filplusCheckNotaryCmd,
filplusSignRemoveDataCapProposal,
},
}
@@ -277,112 +274,3 @@ func checkNotary(ctx context.Context, api v0api.FullNode, vaddr address.Address)
return st.VerifierDataCap(vid)
}
var filplusSignRemoveDataCapProposal = &cli.Command{
Name: "sign-remove-data-cap-proposal",
Usage: "allows a notary to sign a Remove Data Cap Proposal",
Flags: []cli.Flag{
&cli.Int64Flag{
Name: "id",
Usage: "specify the RemoveDataCapProposal ID (will look up on chain if unspecified)",
Required: false,
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 3 {
return fmt.Errorf("must specify three arguments: notary address, client address, and allowance to remove")
}
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return xerrors.Errorf("failed to get full node api: %w", err)
}
defer closer()
ctx := ReqContext(cctx)
act, err := api.StateGetActor(ctx, verifreg.Address, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("failed to get verifreg actor: %w", err)
}
apibs := blockstore.NewAPIBlockstore(api)
store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
st, err := verifreg.Load(store, act)
if err != nil {
return xerrors.Errorf("failed to load verified registry state: %w", err)
}
verifier, err := address.NewFromString(cctx.Args().Get(0))
if err != nil {
return err
}
verifierIdAddr, err := api.StateLookupID(ctx, verifier, types.EmptyTSK)
if err != nil {
return err
}
client, err := address.NewFromString(cctx.Args().Get(1))
if err != nil {
return err
}
clientIdAddr, err := api.StateLookupID(ctx, client, types.EmptyTSK)
if err != nil {
return err
}
allowanceToRemove, err := types.BigFromString(cctx.Args().Get(2))
if err != nil {
return err
}
_, dataCap, err := st.VerifiedClientDataCap(clientIdAddr)
if err != nil {
return xerrors.Errorf("failed to find verified client data cap: %w", err)
}
if dataCap.LessThanEqual(big.Zero()) {
return xerrors.Errorf("client data cap %s is less than amount requested to be removed %s", dataCap.String(), allowanceToRemove.String())
}
found, _, err := checkNotary(ctx, api, verifier)
if err != nil {
return xerrors.Errorf("failed to check notary status: %w", err)
}
if !found {
return xerrors.New("verifier address must be a notary")
}
id := cctx.Uint64("id")
if id == 0 {
_, id, err = st.RemoveDataCapProposalID(verifierIdAddr, clientIdAddr)
if err != nil {
return xerrors.Errorf("failed find remove data cap proposal id: %w", err)
}
}
params := verifreg.RemoveDataCapProposal{
RemovalProposalID: verifreg.RmDcProposalID{ProposalID: id},
DataCapAmount: allowanceToRemove,
VerifiedClient: clientIdAddr,
}
paramBuf := new(bytes.Buffer)
paramBuf.WriteString(verifreg.SignatureDomainSeparation_RemoveDataCap)
err = params.MarshalCBOR(paramBuf)
if err != nil {
return xerrors.Errorf("failed to marshall paramBuf: %w", err)
}
sig, err := api.WalletSign(ctx, verifier, paramBuf.Bytes())
if err != nil {
return xerrors.Errorf("failed to sign message: %w", err)
}
sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)
fmt.Println(hex.EncodeToString(sigBytes))
return nil
},
}
-32
View File
@@ -1,32 +0,0 @@
package cli
import (
"bytes"
"testing"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/mocks"
"github.com/golang/mock/gomock"
ucli "github.com/urfave/cli/v2"
)
// newMockAppWithFullAPI returns a gomock-ed CLI app used for unit tests
// see cli/util/api.go:GetFullNodeAPI for mock API injection
func NewMockAppWithFullAPI(t *testing.T, cmd *ucli.Command) (*ucli.App, *mocks.MockFullNode, *bytes.Buffer, func()) {
app := ucli.NewApp()
app.Commands = ucli.Commands{cmd}
app.Setup()
// create and inject the mock API into app Metadata
ctrl := gomock.NewController(t)
mockFullNode := mocks.NewMockFullNode(ctrl)
var fullNode api.FullNode = mockFullNode
app.Metadata["test-full-api"] = fullNode
// this will only work if the implementation uses the app.Writer,
// if it uses fmt.*, it has to be refactored
buf := &bytes.Buffer{}
app.Writer = buf
return app, mockFullNode, buf, ctrl.Finish
}
+24 -37
View File
@@ -60,8 +60,6 @@ var MpoolPending = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -74,7 +72,7 @@ var MpoolPending = &cli.Command{
if tos := cctx.String("to"); tos != "" {
a, err := address.NewFromString(tos)
if err != nil {
return xerrors.Errorf("given 'to' address %q was invalid: %w", tos, err)
return fmt.Errorf("given 'to' address %q was invalid: %w", tos, err)
}
toa = a
}
@@ -82,7 +80,7 @@ var MpoolPending = &cli.Command{
if froms := cctx.String("from"); froms != "" {
a, err := address.NewFromString(froms)
if err != nil {
return xerrors.Errorf("given 'from' address %q was invalid: %w", froms, err)
return fmt.Errorf("given 'from' address %q was invalid: %w", froms, err)
}
froma = a
}
@@ -121,13 +119,13 @@ var MpoolPending = &cli.Command{
}
if cctx.Bool("cids") {
afmt.Println(msg.Cid())
fmt.Println(msg.Cid())
} else {
out, err := json.MarshalIndent(msg, "", " ")
if err != nil {
return err
}
afmt.Println(string(out))
fmt.Println(string(out))
}
}
@@ -218,8 +216,6 @@ var MpoolStat = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -238,7 +234,6 @@ var MpoolStat = &cli.Command{
currTs := ts
for i := 0; i < cctx.Int("basefee-lookback"); i++ {
currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
if err != nil {
return xerrors.Errorf("walking chain: %w", err)
}
@@ -301,7 +296,7 @@ var MpoolStat = &cli.Command{
for a, bkt := range buckets {
act, err := api.StateGetActor(ctx, a, ts.Key())
if err != nil {
afmt.Printf("%s, err: %s\n", a, err)
fmt.Printf("%s, err: %s\n", a, err)
continue
}
@@ -355,11 +350,11 @@ var MpoolStat = &cli.Command{
total.belowPast += stat.belowPast
total.gasLimit = big.Add(total.gasLimit, stat.gasLimit)
afmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
fmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
}
afmt.Println("-----")
afmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)
fmt.Println("-----")
fmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)
return nil
},
@@ -390,9 +385,8 @@ var MpoolReplaceCmd = &cli.Command{
Usage: "Spend up to X FIL for this message in units of FIL. Previously when flag was `max-fee` units were in attoFIL. Applicable for auto mode",
},
},
ArgsUsage: "<from> <nonce> | <message-cid>",
ArgsUsage: "<from nonce> | <message-cid>",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
@@ -413,14 +407,13 @@ var MpoolReplaceCmd = &cli.Command{
msg, err := api.ChainGetMessage(ctx, mcid)
if err != nil {
return xerrors.Errorf("could not find referenced message: %w", err)
return fmt.Errorf("could not find referenced message: %w", err)
}
from = msg.From
nonce = msg.Nonce
case 2:
arg0 := cctx.Args().Get(0)
f, err := address.NewFromString(arg0)
f, err := address.NewFromString(cctx.Args().Get(0))
if err != nil {
return err
}
@@ -455,7 +448,7 @@ var MpoolReplaceCmd = &cli.Command{
}
if found == nil {
return xerrors.Errorf("no pending message found from %s with nonce %d", from, nonce)
return fmt.Errorf("no pending message found from %s with nonce %d", from, nonce)
}
msg := found.Message
@@ -467,7 +460,7 @@ var MpoolReplaceCmd = &cli.Command{
if cctx.IsSet("fee-limit") {
maxFee, err := types.ParseFIL(cctx.String("fee-limit"))
if err != nil {
return xerrors.Errorf("parsing max-spend: %w", err)
return fmt.Errorf("parsing max-spend: %w", err)
}
mss = &lapi.MessageSendSpec{
MaxFee: abi.TokenAmount(maxFee),
@@ -479,7 +472,7 @@ var MpoolReplaceCmd = &cli.Command{
msg.GasPremium = abi.NewTokenAmount(0)
retm, err := api.GasEstimateMessageGas(ctx, &msg, mss, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("failed to estimate gas values: %w", err)
return fmt.Errorf("failed to estimate gas values: %w", err)
}
msg.GasPremium = big.Max(retm.GasPremium, minRBF)
@@ -496,26 +489,26 @@ var MpoolReplaceCmd = &cli.Command{
}
msg.GasPremium, err = types.BigFromString(cctx.String("gas-premium"))
if err != nil {
return xerrors.Errorf("parsing gas-premium: %w", err)
return fmt.Errorf("parsing gas-premium: %w", err)
}
// TODO: estimate fee cap here
msg.GasFeeCap, err = types.BigFromString(cctx.String("gas-feecap"))
if err != nil {
return xerrors.Errorf("parsing gas-feecap: %w", err)
return fmt.Errorf("parsing gas-feecap: %w", err)
}
}
smsg, err := api.WalletSignMessage(ctx, msg.From, &msg)
if err != nil {
return xerrors.Errorf("failed to sign message: %w", err)
return fmt.Errorf("failed to sign message: %w", err)
}
cid, err := api.MpoolPush(ctx, smsg)
if err != nil {
return xerrors.Errorf("failed to push new message to mempool: %w", err)
return fmt.Errorf("failed to push new message to mempool: %w", err)
}
afmt.Println("new message cid: ", cid)
fmt.Println("new message cid: ", cid)
return nil
},
}
@@ -538,8 +531,6 @@ var MpoolFindCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -557,7 +548,7 @@ var MpoolFindCmd = &cli.Command{
if cctx.IsSet("to") {
a, err := address.NewFromString(cctx.String("to"))
if err != nil {
return xerrors.Errorf("'to' address was invalid: %w", err)
return fmt.Errorf("'to' address was invalid: %w", err)
}
toFilter = a
@@ -566,7 +557,7 @@ var MpoolFindCmd = &cli.Command{
if cctx.IsSet("from") {
a, err := address.NewFromString(cctx.String("from"))
if err != nil {
return xerrors.Errorf("'from' address was invalid: %w", err)
return fmt.Errorf("'from' address was invalid: %w", err)
}
fromFilter = a
@@ -600,7 +591,7 @@ var MpoolFindCmd = &cli.Command{
return err
}
afmt.Println(string(b))
fmt.Println(string(b))
return nil
},
}
@@ -614,8 +605,6 @@ var MpoolConfig = &cli.Command{
return cli.ShowCommandHelp(cctx, cctx.Command.Name)
}
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -635,7 +624,7 @@ var MpoolConfig = &cli.Command{
return err
}
afmt.Println(string(bytes))
fmt.Println(string(bytes))
} else {
cfg := new(types.MpoolConfig)
bytes := []byte(cctx.Args().Get(0))
@@ -662,8 +651,6 @@ var MpoolGasPerfCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -730,7 +717,7 @@ var MpoolGasPerfCmd = &cli.Command{
gasReward := getGasReward(m)
gasPerf := getGasPerf(gasReward, m.Message.GasLimit)
afmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
fmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
}
return nil
-582
View File
@@ -1,582 +0,0 @@
//stm: #cli
package cli
import (
"context"
"fmt"
"testing"
"encoding/json"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
)
func TestStat(t *testing.T) {
t.Run("local", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// add blocks to the chain
first := mock.TipSet(mock.MkBlock(nil, 5, 4))
head := mock.TipSet(mock.MkBlock(first, 15, 7))
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
// mock actor to return for the sender
actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr, toAddr}, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
)
//stm: @CLI_MEMPOOL_STAT_002
err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1", "--local"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), "Nonce past: 1")
})
t.Run("all", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// add blocks to the chain
first := mock.TipSet(mock.MkBlock(nil, 5, 4))
head := mock.TipSet(mock.MkBlock(first, 15, 7))
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
// mock actor to return for the sender
actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
)
//stm: @CLI_MEMPOOL_STAT_001
err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), "Nonce past: 1")
})
}
func TestPending(t *testing.T) {
t.Run("all", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_PENDING_001
err = app.Run([]string{"mpool", "pending", "--cids"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("local", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_PENDING_002
err = app.Run([]string{"mpool", "pending", "--local"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("to", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_PENDING_003
err = app.Run([]string{"mpool", "pending", "--to", sm.Message.To.String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("from", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_PENDING_004
err = app.Run([]string{"mpool", "pending", "--from", sm.Message.From.String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
}
func TestReplace(t *testing.T) {
t.Run("manual", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, &sm.Message).Return(sm, nil),
mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
)
//stm: @CLI_MEMPOOL_REPLACE_002
err = app.Run([]string{"mpool", "replace", "--gas-premium", "1", "--gas-feecap", "100", sm.Cid().String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("auto", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
// gas fee param should be equal to the one passed in the cli invocation (used below)
maxFee := "1000000"
parsedFee, err := types.ParseFIL(maxFee)
if err != nil {
t.Fatal(err)
}
mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
gomock.InOrder(
mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
// use gomock.any to match the message in expected api calls
// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
)
//stm: @CLI_MEMPOOL_REPLACE_002
err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Cid().String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("sender / nonce", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
// gas fee param should be equal to the one passed in the cli invocation (used below)
maxFee := "1000000"
parsedFee, err := types.ParseFIL(maxFee)
if err != nil {
t.Fatal(err)
}
mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
gomock.InOrder(
mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
// use gomock.any to match the message in expected api calls
// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
)
//stm: @CLI_MEMPOOL_REPLACE_001
err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Message.From.String(), fmt.Sprint(sm.Message.Nonce)})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
}
func TestFindMsg(t *testing.T) {
t.Run("from", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_FIND_001
err = app.Run([]string{"mpool", "find", "--from", sm.Message.From.String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("to", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_FIND_002
err = app.Run([]string{"mpool", "find", "--to", sm.Message.To.String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
t.Run("method", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 1, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
)
//stm: @CLI_MEMPOOL_FIND_003
err = app.Run([]string{"mpool", "find", "--method", sm.Message.Method.String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Cid().String())
})
}
func TestGasPerf(t *testing.T) {
t.Run("all", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// add blocks to the chain
first := mock.TipSet(mock.MkBlock(nil, 5, 4))
head := mock.TipSet(mock.MkBlock(first, 15, 7))
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 13, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
)
//stm: @CLI_MEMPOOL_GAS_PERF_002
err = app.Run([]string{"mpool", "gas-perf", "--all", "true"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Message.From.String())
assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
})
t.Run("local", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// add blocks to the chain
first := mock.TipSet(mock.MkBlock(nil, 5, 4))
head := mock.TipSet(mock.MkBlock(first, 15, 7))
// create a signed message to be returned as a pending message
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
sm := mock.MkMessage(senderAddr, toAddr, 13, w)
gomock.InOrder(
mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
)
//stm: @CLI_MEMPOOL_GAS_PERF_001
err = app.Run([]string{"mpool", "gas-perf"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), sm.Message.From.String())
assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
})
}
func TestConfig(t *testing.T) {
t.Run("get", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25}
gomock.InOrder(
mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil),
)
//stm: @CLI_MEMPOOL_CONFIG_001
err = app.Run([]string{"mpool", "config"})
assert.NoError(t, err)
assert.Contains(t, buf.String(), mpoolCfg.PriorityAddrs[0].String())
assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitHigh))
assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitLow))
assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.ReplaceByFeeRatio))
})
t.Run("set", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
if err != nil {
t.Fatal(err)
}
mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33}
gomock.InOrder(
mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil),
)
bytes, err := json.Marshal(mpoolCfg)
if err != nil {
t.Fatal(err)
}
//stm: @CLI_MEMPOOL_CONFIG_002
err = app.Run([]string{"mpool", "config", string(bytes)})
assert.NoError(t, err)
})
}
-210
View File
@@ -36,11 +36,6 @@ var NetCmd = &cli.Command{
NetReachability,
NetBandwidthCmd,
NetBlockCmd,
NetStatCmd,
NetLimitCmd,
NetProtectAdd,
NetProtectRemove,
NetProtectList,
},
}
@@ -611,208 +606,3 @@ var NetBlockListCmd = &cli.Command{
return nil
},
}
var NetStatCmd = &cli.Command{
Name: "stat",
Usage: "Report resource usage for a scope",
ArgsUsage: "scope",
Description: `Report resource usage for a scope.
The scope can be one of the following:
- system -- reports the system aggregate resource usage.
- transient -- reports the transient resource usage.
- svc:<service> -- reports the resource usage of a specific service.
- proto:<proto> -- reports the resource usage of a specific protocol.
- peer:<peer> -- reports the resource usage of a specific peer.
- all -- reports the resource usage for all currently active scopes.
`,
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
args := cctx.Args().Slice()
if len(args) != 1 {
return xerrors.Errorf("must specify exactly one scope")
}
scope := args[0]
result, err := api.NetStat(ctx, scope)
if err != nil {
return err
}
enc := json.NewEncoder(os.Stdout)
return enc.Encode(result)
},
}
var NetLimitCmd = &cli.Command{
Name: "limit",
Usage: "Get or set resource limits for a scope",
ArgsUsage: "scope [limit]",
Description: `Get or set resource limits for a scope.
The scope can be one of the following:
- system -- reports the system aggregate resource usage.
- transient -- reports the transient resource usage.
- svc:<service> -- reports the resource usage of a specific service.
- proto:<proto> -- reports the resource usage of a specific protocol.
- peer:<peer> -- reports the resource usage of a specific peer.
The limit is json-formatted, with the same structure as the limits file.
`,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "set",
Usage: "set the limit for a scope",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
args := cctx.Args().Slice()
if cctx.Bool("set") {
if len(args) != 2 {
return xerrors.Errorf("must specify exactly a scope and a limit")
}
scope := args[0]
limitStr := args[1]
var limit atypes.NetLimit
err := json.Unmarshal([]byte(limitStr), &limit)
if err != nil {
return xerrors.Errorf("error decoding limit: %w", err)
}
return api.NetSetLimit(ctx, scope, limit)
}
if len(args) != 1 {
return xerrors.Errorf("must specify exactly one scope")
}
scope := args[0]
result, err := api.NetLimit(ctx, scope)
if err != nil {
return err
}
enc := json.NewEncoder(os.Stdout)
return enc.Encode(result)
},
}
var NetProtectAdd = &cli.Command{
Name: "protect",
Usage: "Add one or more peer IDs to the list of protected peer connections",
ArgsUsage: "<peer-id> [<peer-id>...]",
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
pids, err := decodePeerIDsFromArgs(cctx)
if err != nil {
return err
}
err = api.NetProtectAdd(ctx, pids)
if err != nil {
return err
}
fmt.Println("added to protected peers:")
for _, pid := range pids {
fmt.Printf(" %s\n", pid)
}
return nil
},
}
var NetProtectRemove = &cli.Command{
Name: "unprotect",
Usage: "Remove one or more peer IDs from the list of protected peer connections.",
ArgsUsage: "<peer-id> [<peer-id>...]",
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
pids, err := decodePeerIDsFromArgs(cctx)
if err != nil {
return err
}
err = api.NetProtectRemove(ctx, pids)
if err != nil {
return err
}
fmt.Printf("removed from protected peers:")
for _, pid := range pids {
fmt.Printf(" %s\n", pid)
}
return nil
},
}
// decodePeerIDsFromArgs decodes all the arguments present in cli.Context.Args as peer.ID.
//
// This function requires at least one argument to be present, and arguments must not be empty
// string. Otherwise, an error is returned.
func decodePeerIDsFromArgs(cctx *cli.Context) ([]peer.ID, error) {
pidArgs := cctx.Args().Slice()
if len(pidArgs) == 0 {
return nil, xerrors.Errorf("must specify at least one peer ID as an argument")
}
var pids []peer.ID
for _, pidStr := range pidArgs {
if pidStr == "" {
return nil, xerrors.Errorf("peer ID must not be empty")
}
pid, err := peer.Decode(pidStr)
if err != nil {
return nil, err
}
pids = append(pids, pid)
}
return pids, nil
}
var NetProtectList = &cli.Command{
Name: "list-protected",
Usage: "List the peer IDs with protected connection.",
Action: func(cctx *cli.Context) error {
api, closer, err := GetAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
pids, err := api.NetProtectList(ctx)
if err != nil {
return err
}
for _, pid := range pids {
fmt.Printf("%s\n", pid)
}
return nil
},
}
+12 -16
View File
@@ -33,8 +33,6 @@ var SyncStatusCmd = &cli.Command{
Name: "status",
Usage: "check sync status",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
apic, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -47,9 +45,9 @@ var SyncStatusCmd = &cli.Command{
return err
}
afmt.Println("sync status:")
fmt.Println("sync status:")
for _, ss := range state.ActiveSyncs {
afmt.Printf("worker %d:\n", ss.WorkerID)
fmt.Printf("worker %d:\n", ss.WorkerID)
var base, target []cid.Cid
var heightDiff int64
var theight abi.ChainEpoch
@@ -64,20 +62,20 @@ var SyncStatusCmd = &cli.Command{
} else {
heightDiff = 0
}
afmt.Printf("\tBase:\t%s\n", base)
afmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
afmt.Printf("\tHeight diff:\t%d\n", heightDiff)
afmt.Printf("\tStage: %s\n", ss.Stage)
afmt.Printf("\tHeight: %d\n", ss.Height)
fmt.Printf("\tBase:\t%s\n", base)
fmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
fmt.Printf("\tHeight diff:\t%d\n", heightDiff)
fmt.Printf("\tStage: %s\n", ss.Stage)
fmt.Printf("\tHeight: %d\n", ss.Height)
if ss.End.IsZero() {
if !ss.Start.IsZero() {
afmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
fmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
}
} else {
afmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
fmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
}
if ss.Stage == api.StageSyncErrored {
afmt.Printf("\tError: %s\n", ss.Message)
fmt.Printf("\tError: %s\n", ss.Message)
}
}
return nil
@@ -170,8 +168,6 @@ var SyncCheckBadCmd = &cli.Command{
Usage: "check if the given block was marked bad, and for what reason",
ArgsUsage: "[blockCid]",
Action: func(cctx *cli.Context) error {
afmt := NewAppFmt(cctx.App)
napi, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -194,11 +190,11 @@ var SyncCheckBadCmd = &cli.Command{
}
if reason == "" {
afmt.Println("block was not marked as bad")
fmt.Println("block was not marked as bad")
return nil
}
afmt.Println(reason)
fmt.Println(reason)
return nil
},
}
-189
View File
@@ -1,189 +0,0 @@
package cli
import (
"context"
"fmt"
"testing"
"time"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
)
func TestSyncStatus(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncStatusCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts1 := mock.TipSet(mock.MkBlock(nil, 0, 0))
ts2 := mock.TipSet(mock.MkBlock(ts1, 0, 0))
start := time.Now()
end := start.Add(time.Minute)
state := &api.SyncState{
ActiveSyncs: []api.ActiveSync{{
WorkerID: 1,
Base: ts1,
Target: ts2,
Stage: api.StageMessages,
Height: abi.ChainEpoch(0),
Start: start,
End: end,
Message: "whatever",
}},
VMApplied: 0,
}
mockApi.EXPECT().SyncState(ctx).Return(state, nil)
//stm: @CLI_SYNC_STATUS_001
err := app.Run([]string{"sync", "status"})
assert.NoError(t, err)
out := buf.String()
// output is plaintext, had to do string matching
assert.Contains(t, out, fmt.Sprintf("Base:\t[%s]", ts1.Blocks()[0].Cid().String()))
assert.Contains(t, out, fmt.Sprintf("Target:\t[%s]", ts2.Blocks()[0].Cid().String()))
assert.Contains(t, out, "Height diff:\t1")
assert.Contains(t, out, "Stage: message sync")
assert.Contains(t, out, "Height: 0")
assert.Contains(t, out, "Elapsed: 1m0s")
}
func TestSyncMarkBad(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncMarkBadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blk := mock.MkBlock(nil, 0, 0)
mockApi.EXPECT().SyncMarkBad(ctx, blk.Cid()).Return(nil)
//stm: @CLI_SYNC_MARK_BAD_001
err := app.Run([]string{"sync", "mark-bad", blk.Cid().String()})
assert.NoError(t, err)
}
func TestSyncUnmarkBad(t *testing.T) {
t.Run("one-block", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blk := mock.MkBlock(nil, 0, 0)
mockApi.EXPECT().SyncUnmarkBad(ctx, blk.Cid()).Return(nil)
//stm: @CLI_SYNC_UNMARK_BAD_001
err := app.Run([]string{"sync", "unmark-bad", blk.Cid().String()})
assert.NoError(t, err)
})
t.Run("all", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mockApi.EXPECT().SyncUnmarkAllBad(ctx).Return(nil)
//stm: @CLI_SYNC_UNMARK_BAD_002
err := app.Run([]string{"sync", "unmark-bad", "-all"})
assert.NoError(t, err)
})
}
func TestSyncCheckBad(t *testing.T) {
t.Run("not-bad", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blk := mock.MkBlock(nil, 0, 0)
mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return("", nil)
//stm: @CLI_SYNC_CHECK_BAD_002
err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), "block was not marked as bad")
})
t.Run("bad", func(t *testing.T) {
app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blk := mock.MkBlock(nil, 0, 0)
reason := "whatever"
mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return(reason, nil)
//stm: @CLI_SYNC_CHECK_BAD_001
err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
assert.NoError(t, err)
assert.Contains(t, buf.String(), reason)
})
}
func TestSyncCheckpoint(t *testing.T) {
t.Run("tipset", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
blk := mock.MkBlock(nil, 0, 0)
ts := mock.TipSet(blk)
gomock.InOrder(
mockApi.EXPECT().ChainGetBlock(ctx, blk.Cid()).Return(blk, nil),
mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
)
//stm: @CLI_SYNC_CHECKPOINT_001
err := app.Run([]string{"sync", "checkpoint", blk.Cid().String()})
assert.NoError(t, err)
})
t.Run("epoch", func(t *testing.T) {
app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
defer done()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
epoch := abi.ChainEpoch(0)
blk := mock.MkBlock(nil, 0, 0)
ts := mock.TipSet(blk)
gomock.InOrder(
mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(ts, nil),
mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
)
//stm: @CLI_SYNC_CHECKPOINT_002
err := app.Run([]string{"sync", "checkpoint", fmt.Sprintf("-epoch=%d", epoch)})
assert.NoError(t, err)
})
}
-5
View File
@@ -223,11 +223,6 @@ func GetCommonAPI(ctx *cli.Context) (api.CommonNet, jsonrpc.ClientCloser, error)
}
func GetFullNodeAPI(ctx *cli.Context) (v0api.FullNode, jsonrpc.ClientCloser, error) {
// use the mocked API in CLI unit tests, see cli/mocks_test.go for mock definition
if mock, ok := ctx.App.Metadata["test-full-api"]; ok {
return &v0api.WrapperV1Full{FullNode: mock.(v1api.FullNode)}, func() {}, nil
}
if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
return &v0api.WrapperV1Full{FullNode: tn.(v1api.FullNode)}, func() {}, nil
}
-7
View File
@@ -276,13 +276,6 @@ var sealBenchCmd = &cli.Command{
if err != nil {
return xerrors.Errorf("failed to run seals: %w", err)
}
for _, s := range extendedSealedSectors {
sealedSectors = append(sealedSectors, proof.SectorInfo{
SealedCID: s.SealedCID,
SectorNumber: s.SectorNumber,
SealProof: s.SealProof,
})
}
} else {
// TODO: implement sbfs.List() and use that for all cases (preexisting sectorbuilder or not)
+32 -80
View File
@@ -5,7 +5,6 @@ import (
"os"
"github.com/fatih/color"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"github.com/filecoin-project/lotus/api"
@@ -22,7 +21,6 @@ var dagstoreCmd = &cli.Command{
dagstoreRecoverShardCmd,
dagstoreInitializeAllCmd,
dagstoreGcCmd,
dagstoreLookupPiecesCmd,
},
}
@@ -54,7 +52,38 @@ var dagstoreListShardsCmd = &cli.Command{
return err
}
return printTableShards(shards)
if len(shards) == 0 {
return nil
}
tw := tablewriter.New(
tablewriter.Col("Key"),
tablewriter.Col("State"),
tablewriter.Col("Error"),
)
colors := map[string]color.Attribute{
"ShardStateAvailable": color.FgGreen,
"ShardStateServing": color.FgBlue,
"ShardStateErrored": color.FgRed,
"ShardStateNew": color.FgYellow,
}
for _, s := range shards {
m := map[string]interface{}{
"Key": s.Key,
"State": func() string {
if c, ok := colors[s.State]; ok {
return color.New(c).Sprint(s.State)
}
return s.State
}(),
"Error": s.Error,
}
tw.Write(m)
}
return tw.Flush(os.Stdout)
},
}
@@ -236,80 +265,3 @@ var dagstoreGcCmd = &cli.Command{
return nil
},
}
func printTableShards(shards []api.DagstoreShardInfo) error {
if len(shards) == 0 {
return nil
}
tw := tablewriter.New(
tablewriter.Col("Key"),
tablewriter.Col("State"),
tablewriter.Col("Error"),
)
colors := map[string]color.Attribute{
"ShardStateAvailable": color.FgGreen,
"ShardStateServing": color.FgBlue,
"ShardStateErrored": color.FgRed,
"ShardStateNew": color.FgYellow,
}
for _, s := range shards {
m := map[string]interface{}{
"Key": s.Key,
"State": func() string {
if c, ok := colors[s.State]; ok {
return color.New(c).Sprint(s.State)
}
return s.State
}(),
"Error": s.Error,
}
tw.Write(m)
}
return tw.Flush(os.Stdout)
}
var dagstoreLookupPiecesCmd = &cli.Command{
Name: "lookup-pieces",
Usage: "Lookup pieces that a given CID belongs to",
ArgsUsage: "<cid>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
},
Action: func(cctx *cli.Context) error {
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
if cctx.NArg() != 1 {
return fmt.Errorf("must provide a CID")
}
cidStr := cctx.Args().First()
cid, err := cid.Parse(cidStr)
if err != nil {
return fmt.Errorf("invalid CID: %w", err)
}
marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
shards, err := marketsApi.DagstoreLookupPieces(ctx, cid)
if err != nil {
return err
}
return printTableShards(shards)
},
}

Some files were not shown because too many files have changed in this diff Show More