Update to the head of go-fil-markets#673

Update to the head of the PR that introduces indexing integration in
`go-fil-markets` so that failure to connect to full node is logged only
instead of crashing markets process.

Relates to:
 - https://github.com/filecoin-project/go-fil-markets/pull/673
 - https://github.com/filecoin-project/index-provider/issues/177

.circleci/config.yml

@@ -44,13 +44,13 @@ commands:
       - restore_cache:
           name: Restore parameters cache
           keys:
-            - 'v26-2k-lotus-params'
+            - 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
       - run:  ./lotus fetch-params 2048
       - save_cache:
           name: Save parameters cache
-          key: 'v26-2k-lotus-params'
+          key: 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
   install_ipfs:
@@ -785,6 +785,11 @@ workflows:
           suite: itest-deals_512mb
           target: "./itests/deals_512mb_test.go"
       
+      - test:
+          name: test-itest-deals_anycid
+          suite: itest-deals_anycid
+          target: "./itests/deals_anycid_test.go"
+      
       - test:
           name: test-itest-deals_concurrent
           suite: itest-deals_concurrent

.circleci/template.yml

@@ -44,13 +44,13 @@ commands:
       - restore_cache:
           name: Restore parameters cache
           keys:
-            - 'v26-2k-lotus-params'
+            - 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
       - run:  ./lotus fetch-params 2048
       - save_cache:
           name: Save parameters cache
-          key: 'v26-2k-lotus-params'
+          key: 'v25-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
   install_ipfs:

CHANGELOG.md

@@ -1,178 +1,4 @@
 # Lotus changelog
-# 1.15.0-rc1 / 2022-02-10
-
-This is the first release candidates of v1.15.0, detailed changelog will be updated later.
-
-- github.com/filecoin-project/lotus:
-  - bump the version to v1.15.0-rc1
-  - chore: build: v1.14.0 -> master ([filecoin-project/lotus#8053](https://github.com/filecoin-project/lotus/pull/8053))
-  - FinalizeReplicaUpdate ([filecoin-project/lotus#8018](https://github.com/filecoin-project/lotus/pull/8018))
-  - splitstore sortless compaction ([filecoin-project/lotus#8008](https://github.com/filecoin-project/lotus/pull/8008))
-  - perf: chain: Make drand logs in daemon less noisy (#7955) ([filecoin-project/lotus#7955](https://github.com/filecoin-project/lotus/pull/7955))
-  - chore: shed: storage stats 2.0 ([filecoin-project/lotus#7941](https://github.com/filecoin-project/lotus/pull/7941))
-  - StartEpochSealingBuffer triggers packing on timer ([filecoin-project/lotus#7905](https://github.com/filecoin-project/lotus/pull/7905))
-  - deps: update go-paramfetch to v0.0.4 ([filecoin-project/lotus#8050](https://github.com/filecoin-project/lotus/pull/8050))
-  - fix: sealing: Stop recovery attempts after fault ([filecoin-project/lotus#8014](https://github.com/filecoin-project/lotus/pull/8014))
-  - chore: update build and deps ([filecoin-project/lotus#8047](https://github.com/filecoin-project/lotus/pull/8047))
-  - update libp2p to v0.18.0-rc4 ([filecoin-project/lotus#8044](https://github.com/filecoin-project/lotus/pull/8044))
-  - update go-libp2p to v0.18.0-rc3 ([filecoin-project/lotus#8022](https://github.com/filecoin-project/lotus/pull/8022))
-  - misc: api: Annotate lotus tests according to listed behaviors ([filecoin-project/lotus#7835](https://github.com/filecoin-project/lotus/pull/7835))
-  - some basic splitstore refactors ([filecoin-project/lotus#7999](https://github.com/filecoin-project/lotus/pull/7999))
-  - feat: mpool: Cache state nonces ([filecoin-project/lotus#8005](https://github.com/filecoin-project/lotus/pull/8005))
-  - chore: sealer: quieten a log ([filecoin-project/lotus#7998](https://github.com/filecoin-project/lotus/pull/7998))
-  - chore: build: make the OhSnap epoch configurable by an envvar for devnets ([filecoin-project/lotus#7995](https://github.com/filecoin-project/lotus/pull/7995))
-  - chore: chain: fix log ([filecoin-project/lotus#7993](https://github.com/filecoin-project/lotus/pull/7993))
-  - tvx: supply network version when extracting messages. ([filecoin-project/lotus#7996](https://github.com/filecoin-project/lotus/pull/7996))
-  - fix: sealer: manager should lock Unsealed for ReplicaUpdate ([filecoin-project/lotus#7986](https://github.com/filecoin-project/lotus/pull/7986))
-  - chore: remove inaccurate comment in sealtasks ([filecoin-project/lotus#7977](https://github.com/filecoin-project/lotus/pull/7977))
-  - update go-libp2p to v0.18.0-rc2 ([filecoin-project/lotus#7984](https://github.com/filecoin-project/lotus/pull/7984))
-  - Fix: sealer: ReplicaUpdate should fetch the correct files ([filecoin-project/lotus#7983](https://github.com/filecoin-project/lotus/pull/7983))
-  - libp2p v0.18.0-rc1 and initial resource manager integration ([filecoin-project/lotus#7969](https://github.com/filecoin-project/lotus/pull/7969))
-  - chore: merge release/v1.14.0 PRs into master ([filecoin-project/lotus#7979](https://github.com/filecoin-project/lotus/pull/7979))
-  - chore: update PR template ([filecoin-project/lotus#7918](https://github.com/filecoin-project/lotus/pull/7918))
-  - Fix: chain: create a new VM for each epoch ([filecoin-project/lotus#7966](https://github.com/filecoin-project/lotus/pull/7966))
-  - feat(deps): update markets stack ([filecoin-project/lotus#7959](https://github.com/filecoin-project/lotus/pull/7959))
-  - Use go-libp2p-connmgr v0.3.1 ([filecoin-project/lotus#7957](https://github.com/filecoin-project/lotus/pull/7957))
-  - feat: #7747 sealing: Adding conf variable for capping number of concurrent unsealing jobs (#7884) ([filecoin-project/lotus#7884](https://github.com/filecoin-project/lotus/pull/7884))
-  - build: release: bump master version to v1.15.0-dev ([filecoin-project/lotus#7922](https://github.com/filecoin-project/lotus/pull/7922))
-  - feat: build: butterfly SNAPNET ([filecoin-project/lotus#7932](https://github.com/filecoin-project/lotus/pull/7932))
-  - correctness fixes for the autobatch blockstore ([filecoin-project/lotus#7940](https://github.com/filecoin-project/lotus/pull/7940))
-  - fix: #7869 sealing: Add more deal expiration checks during PRU pipeline ([filecoin-project/lotus#7871](https://github.com/filecoin-project/lotus/pull/7871))
-  - fix: blockstore: Add missing locks to autobatch::Get() ([filecoin-project/lotus#7939](https://github.com/filecoin-project/lotus/pull/7939))
-  - feat: state: Fast migration for v15 ([filecoin-project/lotus#7933](https://github.com/filecoin-project/lotus/pull/7933))
-  - enhancement: #7891 sectors: disable existing existing cc upgrade path 2 days before the upgrade epoch ([filecoin-project/lotus#7900](https://github.com/filecoin-project/lotus/pull/7900))
-  - chore: deps: Integrate proof v11.0.0 ([filecoin-project/lotus#7923](https://github.com/filecoin-project/lotus/pull/7923))
-  - Snap Deals Lotus Integration: FSM Posting and integration test ([filecoin-project/lotus#7810](https://github.com/filecoin-project/lotus/pull/7810))
-  - chore: deps: Update go-paramfetch ([filecoin-project/lotus#7917](https://github.com/filecoin-project/lotus/pull/7917))
-  - feat: #7880 gas: add gas charge for VerifyReplicaUpdate ([filecoin-project/lotus#7897](https://github.com/filecoin-project/lotus/pull/7897))
-  - misc: build: v1.13.2 back to master ([filecoin-project/lotus#7910](https://github.com/filecoin-project/lotus/pull/7910))
-  - FIX: #7904 CI: generated vectors from v7 for bleeding edge ([filecoin-project/lotus#7903](https://github.com/filecoin-project/lotus/pull/7903))
-  - misc: docs: remove issue number from the pr title ([filecoin-project/lotus#7902](https://github.com/filecoin-project/lotus/pull/7902))
-  - Fix: state: circsuypply calc around null blocks ([filecoin-project/lotus#7890](https://github.com/filecoin-project/lotus/pull/7890))
-  - test vector runner fixes for v6 vectors ([filecoin-project/lotus#7881](https://github.com/filecoin-project/lotus/pull/7881))
-  - Shed: Add a util to send a batch of messages ([filecoin-project/lotus#7667](https://github.com/filecoin-project/lotus/pull/7667))
-  - Snapcraft grade no develgrade ([filecoin-project/lotus#7802](https://github.com/filecoin-project/lotus/pull/7802))
-  - fix: doc generation struct slice example value ([filecoin-project/lotus#7851](https://github.com/filecoin-project/lotus/pull/7851))
-  - fix: returned error not be accept correctly ([filecoin-project/lotus#7852](https://github.com/filecoin-project/lotus/pull/7852))
-  - Add api for transfer diagnostics ([filecoin-project/lotus#7759](https://github.com/filecoin-project/lotus/pull/7759))
-  - Refactor: VM: Remove the NetworkVersionGetter ([filecoin-project/lotus#7818](https://github.com/filecoin-project/lotus/pull/7818))
-  - fix: #7577 markets: When retrying Add Piece, first seek to start of reader ([filecoin-project/lotus#7812](https://github.com/filecoin-project/lotus/pull/7812))
-  - feat: #6017 market: retrieval ask CLI command ([filecoin-project/lotus#7814](https://github.com/filecoin-project/lotus/pull/7814))
-  - misc: n/a sealing: Fix grammatical error in a log warning message ([filecoin-project/lotus#7831](https://github.com/filecoin-project/lotus/pull/7831))
-  - sectors update-state checks if sector exists before changing its state ([filecoin-project/lotus#7762](https://github.com/filecoin-project/lotus/pull/7762))
-  - refactor: state: Move randomness versioning out of the VM ([filecoin-project/lotus#7816](https://github.com/filecoin-project/lotus/pull/7816))
-  - feat(graphsync): allow setting of per-peer incoming requests for miners ([filecoin-project/lotus#7578](https://github.com/filecoin-project/lotus/pull/7578))
-  - updating to new datastore/blockstore code with contexts ([filecoin-project/lotus#7646](https://github.com/filecoin-project/lotus/pull/7646))
-  - Shed: Add a util to list terminated deals ([filecoin-project/lotus#7774](https://github.com/filecoin-project/lotus/pull/7774))
-  - VM: Circ supply should be constant per epoch ([filecoin-project/lotus#7811](https://github.com/filecoin-project/lotus/pull/7811))
-  - chore: create pull_request_template.md ([filecoin-project/lotus#7726](https://github.com/filecoin-project/lotus/pull/7726))
-  - Mempool msg selection should respect block message limits ([filecoin-project/lotus#7321](https://github.com/filecoin-project/lotus/pull/7321))
-  - Merge v15 work into master ([filecoin-project/lotus#7784](https://github.com/filecoin-project/lotus/pull/7784))
-  - Expose EnableGasTracing as an env_var ([filecoin-project/lotus#7750](https://github.com/filecoin-project/lotus/pull/7750))
-  - Minor improvement for OpenTelemetry ([filecoin-project/lotus#7760](https://github.com/filecoin-project/lotus/pull/7760))
-  - Sort lotus-miner retrieval-deals by dealId ([filecoin-project/lotus#7749](https://github.com/filecoin-project/lotus/pull/7749))
-  - Make retrieval even faster ([filecoin-project/lotus#7746](https://github.com/filecoin-project/lotus/pull/7746))
-  - dep/fix 7701 Dependency: update to ipld-legacy to v0.1.1 ([filecoin-project/lotus#7751](https://github.com/filecoin-project/lotus/pull/7751))
-  - Command to list active sector locks ([filecoin-project/lotus#7735](https://github.com/filecoin-project/lotus/pull/7735))
-  - dagstore pieceReader: Always read full in ReadAt ([filecoin-project/lotus#7737](https://github.com/filecoin-project/lotus/pull/7737))
-  - SplitStore: supress compaction near upgrades ([filecoin-project/lotus#7734](https://github.com/filecoin-project/lotus/pull/7734))
-  - Disable appimage ([filecoin-project/lotus#7707](https://github.com/filecoin-project/lotus/pull/7707))
-  - Initial switch to OpenTelemetry ([filecoin-project/lotus#7725](https://github.com/filecoin-project/lotus/pull/7725))
-  - Bump the master version to v1.13.3-dev ([filecoin-project/lotus#7717](https://github.com/filecoin-project/lotus/pull/7717))
-- github.com/filecoin-project/dagstore (v0.4.3 -> v0.4.4):
-  failed to fetch repo
-- github.com/filecoin-project/go-amt-ipld/v4 (null -> v4.0.0):
-  - v4 (#60) ([filecoin-project/go-amt-ipld#60](https://github.com/filecoin-project/go-amt-ipld/pull/60))
-  - Merge pull request from GHSA-3wpc-wmc7-8p5g
-  - add .github/workflows/tagpush.yml
-  - add .github/workflows/release-check.yml
-  - add .github/workflows/releaser.yml
-  - update .github/workflows/go-check.yml
-  - update .github/workflows/go-test.yml
-  - add version.json file
-  - fix: fix bitWidth error message (#58) ([filecoin-project/go-amt-ipld#58](https://github.com/filecoin-project/go-amt-ipld/pull/58))
-  - update .github/workflows/go-check.yml
-  - update .github/workflows/go-test.yml
-  - update .github/workflows/automerge.yml
-- github.com/filecoin-project/go-commp-utils (v0.1.2 -> v0.1.3):
-  - chore(deps): clean up dependencies (#7) ([filecoin-project/go-commp-utils#7](https://github.com/filecoin-project/go-commp-utils/pull/7))
-- github.com/filecoin-project/go-ds-versioning (v0.1.0 -> v0.1.1):
-  - update to context datastores (#3) ([filecoin-project/go-ds-versioning#3](https://github.com/filecoin-project/go-ds-versioning/pull/3))
-  - ci(circle): fix versions (#4) ([filecoin-project/go-ds-versioning#4](https://github.com/filecoin-project/go-ds-versioning/pull/4))
-- github.com/filecoin-project/go-fil-markets (v1.13.4 -> v1.19.0):
-  - release: v1.19.0
-  - Update to go-data-transfer v1.14.0 / go-libp2p v0.18.0-rc1 (#669) ([filecoin-project/go-fil-markets#669](https://github.com/filecoin-project/go-fil-markets/pull/669))
-  - release: v1.18.0 (#671) ([filecoin-project/go-fil-markets#671](https://github.com/filecoin-project/go-fil-markets/pull/671))
-  - revert index provider PR #629 and associated PRs (#670) ([filecoin-project/go-fil-markets#670](https://github.com/filecoin-project/go-fil-markets/pull/670))
-  - log advertisement cid for announcement and update deps (#667) ([filecoin-project/go-fil-markets#667](https://github.com/filecoin-project/go-fil-markets/pull/667))
-  - release: v1.15.0 ([filecoin-project/go-fil-markets#661](https://github.com/filecoin-project/go-fil-markets/pull/661))
-  - retrieve by any CID (not just root CID) and reference provider integration (#629) ([filecoin-project/go-fil-markets#629](https://github.com/filecoin-project/go-fil-markets/pull/629))
-  - release: v1.14.1 ([filecoin-project/go-fil-markets#660](https://github.com/filecoin-project/go-fil-markets/pull/660))
-  - refactor: isolate commp in its own package to avoid exposing pulling filecoin-ffi (#659) ([filecoin-project/go-fil-markets#659](https://github.com/filecoin-project/go-fil-markets/pull/659))
-  - release: v1.14.0 ([filecoin-project/go-fil-markets#658](https://github.com/filecoin-project/go-fil-markets/pull/658))
-  - add SeekStart method to inflator reader (#656) ([filecoin-project/go-fil-markets#656](https://github.com/filecoin-project/go-fil-markets/pull/656))
-  - Merge tag 'v1.13.6'
-  - Timeout provider after waiting a period of time for transfer to restart (#655) ([filecoin-project/go-fil-markets#655](https://github.com/filecoin-project/go-fil-markets/pull/655))
-  - Update to context datastores (#653) ([filecoin-project/go-fil-markets#653](https://github.com/filecoin-project/go-fil-markets/pull/653))
-  - Fix and de-dupe link to spec (#652) ([filecoin-project/go-fil-markets#652](https://github.com/filecoin-project/go-fil-markets/pull/652))
-  - release: v1.13.4 (#651) ([filecoin-project/go-fil-markets#651](https://github.com/filecoin-project/go-fil-markets/pull/651))
-- github.com/filecoin-project/go-paramfetch (v0.0.2 -> v0.0.4):
-  - retry fetching once before erroring ([filecoin-project/go-paramfetch#22](https://github.com/filecoin-project/go-paramfetch/pull/22))
-  - Always verify SnapDeals params ([filecoin-project/go-paramfetch#20](https://github.com/filecoin-project/go-paramfetch/pull/20))
-- github.com/filecoin-project/go-state-types (v0.1.1 -> v0.1.3):
-  - CBOR encode Addr Pairs for keys (#35) ([filecoin-project/go-state-types#35](https://github.com/filecoin-project/go-state-types/pull/35))
-  - Add AddrPairKey (#34) ([filecoin-project/go-state-types#34](https://github.com/filecoin-project/go-state-types/pull/34))
-- github.com/filecoin-project/go-statestore (v0.1.1 -> v0.2.0):
-  - update to context datastores ([filecoin-project/go-statestore#10](https://github.com/filecoin-project/go-statestore/pull/10))
-- github.com/filecoin-project/go-storedcounter (v0.0.0-20200421200003-1c99c62e8a5b -> v0.1.0):
-  - Update go-datastore ([filecoin-project/go-storedcounter#1](https://github.com/filecoin-project/go-storedcounter/pull/1))
-- github.com/filecoin-project/specs-actors/v2 (v2.3.5 -> v2.3.6):
-  - Update to context datastores (#1541) ([filecoin-project/specs-actors#1541](https://github.com/filecoin-project/specs-actors/pull/1541))
-- github.com/filecoin-project/specs-storage (v0.1.1-0.20201105051918-5188d9774506 -> v0.2.0):
-  - FinalizeReplicaUpdate (#22) ([filecoin-project/specs-storage#22](https://github.com/filecoin-project/specs-storage/pull/22))
-  - Add release of sectorkey and replica update ([filecoin-project/specs-storage#20](https://github.com/filecoin-project/specs-storage/pull/20))
-  - Feat/update post gen (#19) ([filecoin-project/specs-storage#19](https://github.com/filecoin-project/specs-storage/pull/19))
-  - Add sector key to ReplicaUpdate iface method (#18) ([filecoin-project/specs-storage#18](https://github.com/filecoin-project/specs-storage/pull/18))
-  - Update for snap deals (#17) ([filecoin-project/specs-storage#17](https://github.com/filecoin-project/specs-storage/pull/17))
-  - Merge remote-tracking branch 'origin/feat/sector-ref'
-
-Contributors
-
-| Contributor | Commits | Lines ± | Files Changed |
-|-------------|---------|---------|---------------|
-| Aayush Rajasekaran | 116 | +12792/-3989 | 579 |
-| zenground0 | 38 | +8894/-1521 | 343 |
-| vyzo | 121 | +4537/-1470 | 258 |
-| zl | 1 | +3725/-309 | 8 |
-| dirkmc | 7 | +1392/-1110 | 61 |
-| Jennifer Wang | 55 | +890/-806 | 121 |
-| Łukasz Magiera | 35 | +1143/-339 | 104 |
-| whyrusleeping | 2 | +483/-585 | 28 |
-| Darko Brdareski | 14 | +725/-276 | 154 |
-| Rod Vagg | 3 | +45/-949 | 12 |
-| hannahhoward | 10 | +481/-133 | 35 |
-| Hannah Howard | 3 | +371/-234 | 22 |
-| c r | 4 | +425/-75 | 18 |
-| TheMenko | 8 | +237/-179 | 17 |
-| Aayush | 15 | +197/-106 | 57 |
-| ZenGround0 | 3 | +263/-25 | 11 |
-| Dirk McCormick | 12 | +188/-40 | 27 |
-| Peter Rabbitson | 3 | +128/-62 | 3 |
-| Steven Allen | 2 | +110/-74 | 2 |
-| Raúl Kripalani | 6 | +128/-49 | 9 |
-| Whyrusleeping | 1 | +76/-70 | 8 |
-| web3-bot | 10 | +99/-17 | 10 |
-| Jiaying Wang | 15 | +58/-23 | 15 |
-| Aarsh Shah | 1 | +23/-19 | 5 |
-| Travis Person | 2 | +2/-20 | 4 |
-| Jakub Sztandera | 3 | +13/-5 | 5 |
-| gstuart | 3 | +12/-1 | 3 |
-| Cory Schwartz | 4 | +5/-6 | 4 |
-| pefish | 1 | +4/-3 | 1 |
-| Colin Kennedy | 1 | +4/-2 | 1 |
-| Rob Quist | 1 | +2/-2 | 1 |
-| shotcollin | 1 | +1/-1 | 1 |
-
 
 # v1.13.2 /  2022-01-09
 

api/api_net.go

@@ -51,6 +51,10 @@ type Net interface {
 	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
 	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
 
+	NetProtectAdd(ctx context.Context, acl []peer.ID) error    //perm:admin
+	NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
+	NetProtectList(ctx context.Context) ([]peer.ID, error)     //perm:read
+
 	// ResourceManager API
 	NetStat(ctx context.Context, scope string) (NetStat, error)          //perm:read
 	NetLimit(ctx context.Context, scope string) (NetLimit, error)        //perm:read

api/api_storage.go

@@ -113,8 +113,6 @@ type StorageMiner interface {
 	// SectorCommitPending returns a list of pending Commit sectors to be sent in the next aggregate message
 	SectorCommitPending(ctx context.Context) ([]abi.SectorID, error) //perm:admin
 	SectorMatchPendingPiecesToOpenSectors(ctx context.Context) error //perm:admin
-	// SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
-	SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
 
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
@@ -132,7 +130,6 @@ type StorageMiner interface {
 	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
 	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
 	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
-	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
 	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
 	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
 	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
@@ -222,6 +219,19 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+	IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
+
+	// DagstorePieceIndexSize returns the size of the piece index.
+	DagstorePieceIndexSize(ctx context.Context) (int64, error) //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read

api/api_worker.go

@@ -39,7 +39,6 @@ type Worker interface {
 	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
 	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
 	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
-	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
 	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
 	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
 	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin

api/mocks/mock_full.go

@@ -1856,6 +1856,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()

api/proxy_gen.go

@@ -593,6 +593,12 @@ type NetStruct struct {
 
 		NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`
 
+		NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
+		NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
+
+		NetProtectRemove func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
 		NetPubsubScores func(p0 context.Context) ([]PubsubScore, error) `perm:"read"`
 
 		NetSetLimit func(p0 context.Context, p1 string, p2 NetLimit) error `perm:"admin"`
@@ -639,6 +645,10 @@ type StorageMinerStruct struct {
 
 		DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`
 
+		DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
+
+		DagstorePieceIndexSize func(p0 context.Context) (int64, error) `perm:"admin"`
+
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
 
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -673,6 +683,10 @@ type StorageMinerStruct struct {
 
 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
 
+		IndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"`
+
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
+
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
 
 		MarketDataTransferDiagnostics func(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) `perm:"write"`
@@ -723,8 +737,6 @@ type StorageMinerStruct struct {
 
 		ReturnFetch func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
 
-		ReturnFinalizeReplicaUpdate func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
-
 		ReturnFinalizeSector func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
 
 		ReturnGenerateSectorKeyFromData func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
@@ -757,8 +769,6 @@ type StorageMinerStruct struct {
 
 		SealingSchedDiag func(p0 context.Context, p1 bool) (interface{}, error) `perm:"admin"`
 
-		SectorAbortUpgrade func(p0 context.Context, p1 abi.SectorNumber) error `perm:"admin"`
-
 		SectorAddPieceToAny func(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) `perm:"admin"`
 
 		SectorCommitFlush func(p0 context.Context) ([]sealiface.CommitBatchRes, error) `perm:"admin"`
@@ -876,8 +886,6 @@ type WorkerStruct struct {
 
 		Fetch func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType, p3 storiface.PathType, p4 storiface.AcquireMode) (storiface.CallID, error) `perm:"admin"`
 
-		FinalizeReplicaUpdate func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
-
 		FinalizeSector func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
 
 		GenerateSectorKeyFromData func(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) `perm:"admin"`
@@ -3670,6 +3678,39 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
 	return *new([]peer.AddrInfo), ErrNotSupported
 }
 
+func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectAdd == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectAdd(p0, p1)
+}
+
+func (s *NetStub) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	if s.Internal.NetProtectList == nil {
+		return *new([]peer.ID), ErrNotSupported
+	}
+	return s.Internal.NetProtectList(p0)
+}
+
+func (s *NetStub) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	return *new([]peer.ID), ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectRemove == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectRemove(p0, p1)
+}
+
+func (s *NetStub) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
 func (s *NetStruct) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	if s.Internal.NetPubsubScores == nil {
 		return *new([]PubsubScore), ErrNotSupported
@@ -3824,6 +3865,28 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
 	return *new([]DagstoreShardInfo), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	if s.Internal.DagstoreLookupPieces == nil {
+		return *new([]DagstoreShardInfo), ErrNotSupported
+	}
+	return s.Internal.DagstoreLookupPieces(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	return *new([]DagstoreShardInfo), ErrNotSupported
+}
+
+func (s *StorageMinerStruct) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
+	if s.Internal.DagstorePieceIndexSize == nil {
+		return 0, ErrNotSupported
+	}
+	return s.Internal.DagstorePieceIndexSize(p0)
+}
+
+func (s *StorageMinerStub) DagstorePieceIndexSize(p0 context.Context) (int64, error) {
+	return 0, ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
 	if s.Internal.DagstoreRecoverShard == nil {
 		return ErrNotSupported
@@ -4011,6 +4074,28 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) IndexerAnnounceAllDeals(p0 context.Context) error {
+	if s.Internal.IndexerAnnounceAllDeals == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceAllDeals(p0)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceAllDeals(p0 context.Context) error {
+	return ErrNotSupported
+}
+
+func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.IndexerAnnounceDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
 	if s.Internal.MarketCancelDataTransfer == nil {
 		return ErrNotSupported
@@ -4286,17 +4371,6 @@ func (s *StorageMinerStub) ReturnFetch(p0 context.Context, p1 storiface.CallID,
 	return ErrNotSupported
 }
 
-func (s *StorageMinerStruct) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
-	if s.Internal.ReturnFinalizeReplicaUpdate == nil {
-		return ErrNotSupported
-	}
-	return s.Internal.ReturnFinalizeReplicaUpdate(p0, p1, p2)
-}
-
-func (s *StorageMinerStub) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
-	return ErrNotSupported
-}
-
 func (s *StorageMinerStruct) ReturnFinalizeSector(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
 	if s.Internal.ReturnFinalizeSector == nil {
 		return ErrNotSupported
@@ -4473,17 +4547,6 @@ func (s *StorageMinerStub) SealingSchedDiag(p0 context.Context, p1 bool) (interf
 	return nil, ErrNotSupported
 }
 
-func (s *StorageMinerStruct) SectorAbortUpgrade(p0 context.Context, p1 abi.SectorNumber) error {
-	if s.Internal.SectorAbortUpgrade == nil {
-		return ErrNotSupported
-	}
-	return s.Internal.SectorAbortUpgrade(p0, p1)
-}
-
-func (s *StorageMinerStub) SectorAbortUpgrade(p0 context.Context, p1 abi.SectorNumber) error {
-	return ErrNotSupported
-}
-
 func (s *StorageMinerStruct) SectorAddPieceToAny(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storage.Data, p3 PieceDealInfo) (SectorOffset, error) {
 	if s.Internal.SectorAddPieceToAny == nil {
 		return *new(SectorOffset), ErrNotSupported
@@ -5034,17 +5097,6 @@ func (s *WorkerStub) Fetch(p0 context.Context, p1 storage.SectorRef, p2 storifac
 	return *new(storiface.CallID), ErrNotSupported
 }
 
-func (s *WorkerStruct) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
-	if s.Internal.FinalizeReplicaUpdate == nil {
-		return *new(storiface.CallID), ErrNotSupported
-	}
-	return s.Internal.FinalizeReplicaUpdate(p0, p1, p2)
-}
-
-func (s *WorkerStub) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
-	return *new(storiface.CallID), ErrNotSupported
-}
-
 func (s *WorkerStruct) FinalizeSector(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
 	if s.Internal.FinalizeSector == nil {
 		return *new(storiface.CallID), ErrNotSupported

api/v0api/v0mocks/mock_full.go

@@ -1769,6 +1769,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()

blockstore/splitstore/README.md

@@ -49,11 +49,10 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
   blockstore and discards writes; this is necessary to support syncing from a snapshot.
 - `MarkSetType` -- specifies the type of markset to use during compaction.
   The markset is the data structure used by compaction/gc to track live objects.
-  The default value is "badger", which will use a disk backed markset using badger.
-  If you have a lot of memory (48G or more) you can also use "map", which will use
-  an in memory markset, speeding up compaction at the cost of higher memory usage.
-  Note: If you are using a VPS with a network volume, you need to provision at least
-  3000 IOPs with the badger markset.
+  The default value is `"map"`, which will use an in-memory map; if you are limited
+  in memory (or indeed see compaction run out of memory), you can also specify
+  `"badger"` which will use an disk backed markset, using badger. This will use
+  much less memory, but will also make compaction slower.
 - `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
   finalities maintained by default, to maintain messages and message receipts in the
   hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -106,12 +105,6 @@ Compaction works transactionally with the following algorithm:
   - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
 - We then end the transaction and compact/gc the hotstore.
 
-As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
-modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
-memory usage; in fact, when using badger as the markset compaction uses very little memory, and
-it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
-memory during compaction.
-
 ## Garbage Collection
 
 TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)

blockstore/splitstore/checkpoint.go

@@ -1,118 +0,0 @@
-package splitstore
-
-import (
-	"bufio"
-	"io"
-	"os"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-	mh "github.com/multiformats/go-multihash"
-)
-
-type Checkpoint struct {
-	file *os.File
-	buf  *bufio.Writer
-}
-
-func NewCheckpoint(path string) (*Checkpoint, error) {
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating checkpoint: %w", err)
-	}
-	buf := bufio.NewWriter(file)
-
-	return &Checkpoint{
-		file: file,
-		buf:  buf,
-	}, nil
-}
-
-func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
-	filein, err := os.Open(path)
-	if err != nil {
-		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
-	}
-	defer filein.Close() //nolint:errcheck
-
-	bufin := bufio.NewReader(filein)
-	start, err := readRawCid(bufin, nil)
-	if err != nil && err != io.EOF {
-		return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
-	}
-
-	fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
-	if err != nil {
-		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
-	}
-	bufout := bufio.NewWriter(fileout)
-
-	return &Checkpoint{
-		file: fileout,
-		buf:  bufout,
-	}, start, nil
-}
-
-func (cp *Checkpoint) Set(c cid.Cid) error {
-	if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
-		return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
-	}
-
-	if err := writeRawCid(cp.buf, c, true); err != nil {
-		return xerrors.Errorf("error writing cid to checkpoint: %w", err)
-	}
-
-	return nil
-}
-
-func (cp *Checkpoint) Close() error {
-	if cp.file == nil {
-		return nil
-	}
-
-	err := cp.file.Close()
-	cp.file = nil
-	cp.buf = nil
-
-	return err
-}
-
-func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
-	sz, err := buf.ReadByte()
-	if err != nil {
-		return cid.Undef, err // don't wrap EOF as it is not an error here
-	}
-
-	if hbuf == nil {
-		hbuf = make([]byte, int(sz))
-	} else {
-		hbuf = hbuf[:int(sz)]
-	}
-
-	if _, err := io.ReadFull(buf, hbuf); err != nil {
-		return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
-	}
-
-	hash, err := mh.Cast(hbuf)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
-	}
-
-	return cid.NewCidV1(cid.Raw, hash), nil
-}
-
-func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
-	hash := c.Hash()
-	if err := buf.WriteByte(byte(len(hash))); err != nil {
-		return err
-	}
-	if _, err := buf.Write(hash); err != nil {
-		return err
-	}
-	if flush {
-		return buf.Flush()
-	}
-
-	return nil
-}

blockstore/splitstore/checkpoint_test.go

@@ -1,147 +0,0 @@
-package splitstore
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-func TestCheckpoint(t *testing.T) {
-	dir, err := ioutil.TempDir("", "checkpoint.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(dir)
-	})
-
-	path := filepath.Join(dir, "checkpoint")
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	cp, err := NewCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Set(k1); err != nil {
-		t.Fatal(err)
-	}
-	if err := cp.Set(k2); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cp, start, err := OpenCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !start.Equals(k2) {
-		t.Fatalf("expected start to be %s; got %s", k2, start)
-	}
-
-	if err := cp.Set(k3); err != nil {
-		t.Fatal(err)
-	}
-	if err := cp.Set(k4); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cp, start, err = OpenCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !start.Equals(k4) {
-		t.Fatalf("expected start to be %s; got %s", k4, start)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	// also test correct operation with an empty checkpoint
-	cp, err = NewCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cp, start, err = OpenCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if start.Defined() {
-		t.Fatal("expected start to be undefined")
-	}
-
-	if err := cp.Set(k1); err != nil {
-		t.Fatal(err)
-	}
-	if err := cp.Set(k2); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cp, start, err = OpenCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !start.Equals(k2) {
-		t.Fatalf("expected start to be %s; got %s", k2, start)
-	}
-
-	if err := cp.Set(k3); err != nil {
-		t.Fatal(err)
-	}
-	if err := cp.Set(k4); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cp, start, err = OpenCheckpoint(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !start.Equals(k4) {
-		t.Fatalf("expected start to be %s; got %s", k4, start)
-	}
-
-	if err := cp.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-}

blockstore/splitstore/coldset.go

@@ -1,102 +0,0 @@
-package splitstore
-
-import (
-	"bufio"
-	"io"
-	"os"
-
-	"golang.org/x/xerrors"
-
-	cid "github.com/ipfs/go-cid"
-)
-
-type ColdSetWriter struct {
-	file *os.File
-	buf  *bufio.Writer
-}
-
-type ColdSetReader struct {
-	file *os.File
-	buf  *bufio.Reader
-}
-
-func NewColdSetWriter(path string) (*ColdSetWriter, error) {
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating coldset: %w", err)
-	}
-	buf := bufio.NewWriter(file)
-
-	return &ColdSetWriter{
-		file: file,
-		buf:  buf,
-	}, nil
-}
-
-func NewColdSetReader(path string) (*ColdSetReader, error) {
-	file, err := os.Open(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error opening coldset: %w", err)
-	}
-	buf := bufio.NewReader(file)
-
-	return &ColdSetReader{
-		file: file,
-		buf:  buf,
-	}, nil
-}
-
-func (s *ColdSetWriter) Write(c cid.Cid) error {
-	return writeRawCid(s.buf, c, false)
-}
-
-func (s *ColdSetWriter) Close() error {
-	if s.file == nil {
-		return nil
-	}
-
-	err1 := s.buf.Flush()
-	err2 := s.file.Close()
-	s.buf = nil
-	s.file = nil
-
-	if err1 != nil {
-		return err1
-	}
-	return err2
-}
-
-func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
-	hbuf := make([]byte, 256)
-	for {
-		next, err := readRawCid(s.buf, hbuf)
-		if err != nil {
-			if err == io.EOF {
-				return nil
-			}
-
-			return xerrors.Errorf("error reading coldset: %w", err)
-		}
-
-		if err := f(next); err != nil {
-			return err
-		}
-	}
-}
-
-func (s *ColdSetReader) Reset() error {
-	_, err := s.file.Seek(0, io.SeekStart)
-	return err
-}
-
-func (s *ColdSetReader) Close() error {
-	if s.file == nil {
-		return nil
-	}
-
-	err := s.file.Close()
-	s.file = nil
-	s.buf = nil
-
-	return err
-}

blockstore/splitstore/coldset_test.go

@@ -1,99 +0,0 @@
-package splitstore
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-)
-
-func TestColdSet(t *testing.T) {
-	dir, err := ioutil.TempDir("", "coldset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(dir)
-	})
-
-	path := filepath.Join(dir, "coldset")
-
-	makeCid := func(i int) cid.Cid {
-		h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	const count = 1000
-	cids := make([]cid.Cid, 0, count)
-	for i := 0; i < count; i++ {
-		cids = append(cids, makeCid(i))
-	}
-
-	cw, err := NewColdSetWriter(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, c := range cids {
-		if err := cw.Write(c); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if err := cw.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	cr, err := NewColdSetReader(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	index := 0
-	err = cr.ForEach(func(c cid.Cid) error {
-		if index >= count {
-			t.Fatal("too many cids")
-		}
-
-		if !c.Equals(cids[index]) {
-			t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
-		}
-
-		index++
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := cr.Reset(); err != nil {
-		t.Fatal(err)
-	}
-
-	index = 0
-	err = cr.ForEach(func(c cid.Cid) error {
-		if index >= count {
-			t.Fatal("too many cids")
-		}
-
-		if !c.Equals(cids[index]) {
-			t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
-		}
-
-		index++
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-}

blockstore/splitstore/markset.go

@@ -14,24 +14,15 @@ var errMarkSetClosed = errors.New("markset closed")
 type MarkSet interface {
 	ObjectVisitor
 	Mark(cid.Cid) error
-	MarkMany([]cid.Cid) error
 	Has(cid.Cid) (bool, error)
 	Close() error
-
-	// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
-	// of abnormal termination during the critical section span.
-	BeginCriticalSection() error
-	// EndCriticalSection ends the critical section span.
-	EndCriticalSection()
 }
 
 type MarkSetEnv interface {
-	// New creates a new markset within the environment.
-	// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
+	// Create creates a new markset within the environment.
+	// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
 	// sizeHint is a hint about the expected size of the markset
-	New(name string, sizeHint int64) (MarkSet, error)
-	// Recover recovers an existing markset persisted on-disk.
-	Recover(name string) (MarkSet, error)
+	Create(name string, sizeHint int64) (MarkSet, error)
 	// Close closes the markset
 	Close() error
 }
@@ -39,7 +30,7 @@ type MarkSetEnv interface {
 func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
 	switch mtype {
 	case "map":
-		return NewMapMarkSetEnv(path)
+		return NewMapMarkSetEnv()
 	case "badger":
 		return NewBadgerMarkSetEnv(path)
 	default:

blockstore/splitstore/markset_badger.go

@@ -3,7 +3,6 @@ package splitstore
 import (
 	"os"
 	"path/filepath"
-	"runtime"
 	"sync"
 
 	"golang.org/x/xerrors"
@@ -29,7 +28,6 @@ type BadgerMarkSet struct {
 	writers int
 	seqno   int
 	version int
-	persist bool
 
 	db   *badger.DB
 	path string
@@ -49,10 +47,11 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
 	return &BadgerMarkSetEnv{path: msPath}, nil
 }
 
-func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
+func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
+	name += ".tmp"
 	path := filepath.Join(e.path, name)
 
-	db, err := openBadgerDB(path, false)
+	db, err := openTransientBadgerDB(path)
 	if err != nil {
 		return nil, xerrors.Errorf("error creating badger db: %w", err)
 	}
@@ -68,72 +67,8 @@ func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
 	return ms, nil
 }
 
-func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
-	path := filepath.Join(e.path, name)
-
-	if _, err := os.Stat(path); err != nil {
-		return nil, xerrors.Errorf("error stating badger db path: %w", err)
-	}
-
-	db, err := openBadgerDB(path, true)
-	if err != nil {
-		return nil, xerrors.Errorf("error creating badger db: %w", err)
-	}
-
-	ms := &BadgerMarkSet{
-		pend:    make(map[string]struct{}),
-		writing: make(map[int]map[string]struct{}),
-		db:      db,
-		path:    path,
-		persist: true,
-	}
-	ms.cond.L = &ms.mx
-
-	return ms, nil
-}
-
 func (e *BadgerMarkSetEnv) Close() error {
-	return nil
-}
-
-func (s *BadgerMarkSet) BeginCriticalSection() error {
-	s.mx.Lock()
-
-	if s.persist {
-		s.mx.Unlock()
-		return nil
-	}
-
-	var write bool
-	var seqno int
-	if len(s.pend) > 0 {
-		write = true
-		seqno = s.nextBatch()
-	}
-
-	s.persist = true
-	s.mx.Unlock()
-
-	if write {
-		// all writes sync once perist is true
-		return s.write(seqno)
-	}
-
-	// wait for any pending writes and sync
-	s.mx.Lock()
-	for s.writers > 0 {
-		s.cond.Wait()
-	}
-	s.mx.Unlock()
-
-	return s.db.Sync()
-}
-
-func (s *BadgerMarkSet) EndCriticalSection() {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	s.persist = false
+	return os.RemoveAll(e.path)
 }
 
 func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -153,23 +88,6 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
 	return nil
 }
 
-func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
-	s.mx.Lock()
-	if s.pend == nil {
-		s.mx.Unlock()
-		return errMarkSetClosed
-	}
-
-	write, seqno := s.putMany(batch)
-	s.mx.Unlock()
-
-	if write {
-		return s.write(seqno)
-	}
-
-	return nil
-}
-
 func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
 	s.mx.RLock()
 	defer s.mx.RUnlock()
@@ -275,34 +193,16 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
 // writer holds the exclusive lock
 func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
 	s.pend[key] = struct{}{}
-	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
+	if len(s.pend) < badgerMarkSetBatchSize {
 		return false, 0
 	}
 
-	seqno = s.nextBatch()
-	return true, seqno
-}
-
-func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
-	for _, c := range batch {
-		key := string(c.Hash())
-		s.pend[key] = struct{}{}
-	}
-
-	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
-		return false, 0
-	}
-
-	seqno = s.nextBatch()
-	return true, seqno
-}
-
-func (s *BadgerMarkSet) nextBatch() int {
-	seqno := s.seqno
+	seqno = s.seqno
 	s.seqno++
 	s.writing[seqno] = s.pend
 	s.pend = make(map[string]struct{})
-	return seqno
+
+	return true, seqno
 }
 
 func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -347,14 +247,6 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
 	}
 
-	s.mx.RLock()
-	persist := s.persist
-	s.mx.RUnlock()
-
-	if persist {
-		return s.db.Sync()
-	}
-
 	return nil
 }
 
@@ -374,29 +266,26 @@ func (s *BadgerMarkSet) Close() error {
 	db := s.db
 	s.db = nil
 
-	return closeBadgerDB(db, s.path, s.persist)
+	return closeTransientBadgerDB(db, s.path)
 }
 
-func openBadgerDB(path string, recover bool) (*badger.DB, error) {
-	// if it is not a recovery, clean up first
-	if !recover {
-		err := os.RemoveAll(path)
-		if err != nil {
-			return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-		}
+func (s *BadgerMarkSet) SetConcurrent() {}
 
-		err = os.MkdirAll(path, 0755) //nolint:gosec
-		if err != nil {
-			return nil, xerrors.Errorf("error creating markset directory: %w", err)
-		}
+func openTransientBadgerDB(path string) (*badger.DB, error) {
+	// clean up first
+	err := os.RemoveAll(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
+	}
+
+	err = os.MkdirAll(path, 0755) //nolint:gosec
+	if err != nil {
+		return nil, xerrors.Errorf("error creating markset directory: %w", err)
 	}
 
 	opts := badger.DefaultOptions(path)
-	// we manually sync when we are in critical section
 	opts.SyncWrites = false
-	// no need to do that
 	opts.CompactL0OnClose = false
-	// we store hashes, not much to gain by compression
 	opts.Compression = options.None
 	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
 	//       between the system blockstore and the markset.
@@ -405,15 +294,6 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
 	//       exceeded 1GB in size.
 	opts.TableLoadingMode = options.FileIO
 	opts.ValueLogLoadingMode = options.FileIO
-	// We increase the number of L0 tables before compaction to make it unlikely to
-	// be necessary.
-	opts.NumLevelZeroTables = 20      // default is 5
-	opts.NumLevelZeroTablesStall = 30 // default is 10
-	// increase the number of compactors from default 2 so that if we ever have to
-	// compact, it is fast
-	if runtime.NumCPU()/2 > opts.NumCompactors {
-		opts.NumCompactors = runtime.NumCPU() / 2
-	}
 	opts.Logger = &badgerLogger{
 		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
 		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -422,16 +302,12 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) {
 	return badger.Open(opts)
 }
 
-func closeBadgerDB(db *badger.DB, path string, persist bool) error {
+func closeTransientBadgerDB(db *badger.DB, path string) error {
 	err := db.Close()
 	if err != nil {
 		return xerrors.Errorf("error closing badger markset: %w", err)
 	}
 
-	if persist {
-		return nil
-	}
-
 	err = os.RemoveAll(path)
 	if err != nil {
 		return xerrors.Errorf("error deleting badger markset: %w", err)

blockstore/splitstore/markset_map.go

@@ -1,104 +1,37 @@
 package splitstore
 
 import (
-	"bufio"
-	"io"
-	"os"
-	"path/filepath"
 	"sync"
 
-	"golang.org/x/xerrors"
-
 	cid "github.com/ipfs/go-cid"
 )
 
-type MapMarkSetEnv struct {
-	path string
-}
+type MapMarkSetEnv struct{}
 
 var _ MarkSetEnv = (*MapMarkSetEnv)(nil)
 
 type MapMarkSet struct {
 	mx  sync.RWMutex
 	set map[string]struct{}
-
-	persist bool
-	file    *os.File
-	buf     *bufio.Writer
-
-	path string
 }
 
 var _ MarkSet = (*MapMarkSet)(nil)
 
-func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
-	msPath := filepath.Join(path, "markset.map")
-	err := os.MkdirAll(msPath, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
-	}
-
-	return &MapMarkSetEnv{path: msPath}, nil
+func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
+	return &MapMarkSetEnv{}, nil
 }
 
-func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
-	path := filepath.Join(e.path, name)
+func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
 	return &MapMarkSet{
-		set:  make(map[string]struct{}, sizeHint),
-		path: path,
+		set: make(map[string]struct{}, sizeHint),
 	}, nil
 }
 
-func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
-	path := filepath.Join(e.path, name)
-	s := &MapMarkSet{
-		set:  make(map[string]struct{}),
-		path: path,
-	}
-
-	in, err := os.Open(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error opening markset file for read: %w", err)
-	}
-	defer in.Close() //nolint:errcheck
-
-	// wrap a buffered reader to make this faster
-	buf := bufio.NewReader(in)
-	for {
-		var sz byte
-		if sz, err = buf.ReadByte(); err != nil {
-			break
-		}
-
-		key := make([]byte, int(sz))
-		if _, err = io.ReadFull(buf, key); err != nil {
-			break
-		}
-
-		s.set[string(key)] = struct{}{}
-	}
-
-	if err != io.EOF {
-		return nil, xerrors.Errorf("error reading markset file: %w", err)
-	}
-
-	file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
-	if err != nil {
-		return nil, xerrors.Errorf("error opening markset file for write: %w", err)
-	}
-
-	s.persist = true
-	s.file = file
-	s.buf = bufio.NewWriter(file)
-
-	return s, nil
-}
-
 func (e *MapMarkSetEnv) Close() error {
 	return nil
 }
 
-func (s *MapMarkSet) BeginCriticalSection() error {
+func (s *MapMarkSet) Mark(cid cid.Cid) error {
 	s.mx.Lock()
 	defer s.mx.Unlock()
 
@@ -106,104 +39,7 @@ func (s *MapMarkSet) BeginCriticalSection() error {
 		return errMarkSetClosed
 	}
 
-	if s.persist {
-		return nil
-	}
-
-	file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return xerrors.Errorf("error opening markset file: %w", err)
-	}
-
-	// wrap a buffered writer to make this faster
-	s.buf = bufio.NewWriter(file)
-	for key := range s.set {
-		if err := s.writeKey([]byte(key), false); err != nil {
-			_ = file.Close()
-			s.buf = nil
-			return err
-		}
-	}
-	if err := s.buf.Flush(); err != nil {
-		_ = file.Close()
-		s.buf = nil
-		return xerrors.Errorf("error flushing markset file buffer: %w", err)
-	}
-
-	s.file = file
-	s.persist = true
-
-	return nil
-}
-
-func (s *MapMarkSet) EndCriticalSection() {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	if !s.persist {
-		return
-	}
-
-	_ = s.file.Close()
-	_ = os.Remove(s.path)
-	s.file = nil
-	s.buf = nil
-	s.persist = false
-}
-
-func (s *MapMarkSet) Mark(c cid.Cid) error {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	if s.set == nil {
-		return errMarkSetClosed
-	}
-
-	hash := c.Hash()
-	s.set[string(hash)] = struct{}{}
-
-	if s.persist {
-		if err := s.writeKey(hash, true); err != nil {
-			return err
-		}
-
-		if err := s.file.Sync(); err != nil {
-			return xerrors.Errorf("error syncing markset: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
-	s.mx.Lock()
-	defer s.mx.Unlock()
-
-	if s.set == nil {
-		return errMarkSetClosed
-	}
-
-	for _, c := range batch {
-		hash := c.Hash()
-		s.set[string(hash)] = struct{}{}
-
-		if s.persist {
-			if err := s.writeKey(hash, false); err != nil {
-				return err
-			}
-		}
-	}
-
-	if s.persist {
-		if err := s.buf.Flush(); err != nil {
-			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
-		}
-
-		if err := s.file.Sync(); err != nil {
-			return xerrors.Errorf("error syncing markset: %w", err)
-		}
-	}
-
+	s.set[string(cid.Hash())] = struct{}{}
 	return nil
 }
 
@@ -227,23 +63,12 @@ func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
 		return false, errMarkSetClosed
 	}
 
-	hash := c.Hash()
-	key := string(hash)
+	key := string(c.Hash())
 	if _, ok := s.set[key]; ok {
 		return false, nil
 	}
 
 	s.set[key] = struct{}{}
-
-	if s.persist {
-		if err := s.writeKey(hash, true); err != nil {
-			return false, err
-		}
-		if err := s.file.Sync(); err != nil {
-			return false, xerrors.Errorf("error syncing markset: %w", err)
-		}
-	}
-
 	return true, nil
 }
 
@@ -251,39 +76,6 @@ func (s *MapMarkSet) Close() error {
 	s.mx.Lock()
 	defer s.mx.Unlock()
 
-	if s.set == nil {
-		return nil
-	}
-
 	s.set = nil
-
-	if s.file != nil {
-		if err := s.file.Close(); err != nil {
-			log.Warnf("error closing markset file: %s", err)
-		}
-
-		if !s.persist {
-			if err := os.Remove(s.path); err != nil {
-				log.Warnf("error removing markset file: %s", err)
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
-	if err := s.buf.WriteByte(byte(len(k))); err != nil {
-		return xerrors.Errorf("error writing markset key length to disk: %w", err)
-	}
-	if _, err := s.buf.Write(k); err != nil {
-		return xerrors.Errorf("error writing markset key to disk: %w", err)
-	}
-	if flush {
-		if err := s.buf.Flush(); err != nil {
-			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
-		}
-	}
-
 	return nil
 }

blockstore/splitstore/markset_test.go

@@ -11,10 +11,7 @@ import (
 
 func TestMapMarkSet(t *testing.T) {
 	testMarkSet(t, "map")
-	testMarkSetRecovery(t, "map")
-	testMarkSetMarkMany(t, "map")
 	testMarkSetVisitor(t, "map")
-	testMarkSetVisitorRecovery(t, "map")
 }
 
 func TestBadgerMarkSet(t *testing.T) {
@@ -24,13 +21,12 @@ func TestBadgerMarkSet(t *testing.T) {
 		badgerMarkSetBatchSize = bs
 	})
 	testMarkSet(t, "badger")
-	testMarkSetRecovery(t, "badger")
-	testMarkSetMarkMany(t, "badger")
 	testMarkSetVisitor(t, "badger")
-	testMarkSetVisitorRecovery(t, "badger")
 }
 
 func testMarkSet(t *testing.T, lsType string) {
+	t.Helper()
+
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -46,12 +42,12 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck
 
-	hotSet, err := env.New("hot", 0)
+	hotSet, err := env.Create("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	coldSet, err := env.New("cold", 0)
+	coldSet, err := env.Create("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,7 +62,6 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 
 	mustHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -78,7 +73,6 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 
 	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -120,12 +114,12 @@ func testMarkSet(t *testing.T, lsType string) {
 		t.Fatal(err)
 	}
 
-	hotSet, err = env.New("hot", 0)
+	hotSet, err = env.Create("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	coldSet, err = env.New("cold", 0)
+	coldSet, err = env.Create("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -156,6 +150,8 @@ func testMarkSet(t *testing.T, lsType string) {
 }
 
 func testMarkSetVisitor(t *testing.T, lsType string) {
+	t.Helper()
+
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -171,7 +167,7 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck
 
-	visitor, err := env.New("test", 0)
+	visitor, err := env.Create("test", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -223,322 +219,3 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	mustNotVisit(visitor, k3)
 	mustNotVisit(visitor, k4)
 }
-
-func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	visitor, err := env.New("test", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer visitor.Close() //nolint:errcheck
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustVisit := func(v ObjectVisitor, cid cid.Cid) {
-		visit, err := v.Visit(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !visit {
-			t.Fatal("object should be visited")
-		}
-	}
-
-	mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
-		visit, err := v.Visit(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if visit {
-			t.Fatal("unexpected visit")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	mustVisit(visitor, k1)
-	mustVisit(visitor, k2)
-
-	if err := visitor.BeginCriticalSection(); err != nil {
-		t.Fatal(err)
-	}
-
-	mustVisit(visitor, k3)
-	mustVisit(visitor, k4)
-
-	mustNotVisit(visitor, k1)
-	mustNotVisit(visitor, k2)
-	mustNotVisit(visitor, k3)
-	mustNotVisit(visitor, k4)
-
-	if err := visitor.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	visitor, err = env.Recover("test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	mustNotVisit(visitor, k1)
-	mustNotVisit(visitor, k2)
-	mustNotVisit(visitor, k3)
-	mustNotVisit(visitor, k4)
-
-	visitor.EndCriticalSection()
-
-	if err := visitor.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = env.Recover("test")
-	if err == nil {
-		t.Fatal("expected recovery to fail")
-	}
-}
-
-func testMarkSetRecovery(t *testing.T, lsType string) {
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	markSet, err := env.New("test", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !has {
-			t.Fatal("mark not found")
-		}
-	}
-
-	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if has {
-			t.Fatal("unexpected mark")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	if err := markSet.Mark(k1); err != nil {
-		t.Fatal(err)
-	}
-	if err := markSet.Mark(k2); err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustNotHave(markSet, k3)
-	mustNotHave(markSet, k4)
-
-	if err := markSet.BeginCriticalSection(); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := markSet.Mark(k3); err != nil {
-		t.Fatal(err)
-	}
-	if err := markSet.Mark(k4); err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustHave(markSet, k3)
-	mustHave(markSet, k4)
-
-	if err := markSet.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	markSet, err = env.Recover("test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustHave(markSet, k3)
-	mustHave(markSet, k4)
-
-	markSet.EndCriticalSection()
-
-	if err := markSet.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = env.Recover("test")
-	if err == nil {
-		t.Fatal("expected recovery to fail")
-	}
-}
-
-func testMarkSetMarkMany(t *testing.T, lsType string) {
-	path, err := ioutil.TempDir("", "markset.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
-
-	env, err := OpenMarkSetEnv(path, lsType)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer env.Close() //nolint:errcheck
-
-	markSet, err := env.New("test", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	makeCid := func(key string) cid.Cid {
-		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		return cid.NewCidV1(cid.Raw, h)
-	}
-
-	mustHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if !has {
-			t.Fatal("mark not found")
-		}
-	}
-
-	mustNotHave := func(s MarkSet, cid cid.Cid) {
-		t.Helper()
-		has, err := s.Has(cid)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if has {
-			t.Fatal("unexpected mark")
-		}
-	}
-
-	k1 := makeCid("a")
-	k2 := makeCid("b")
-	k3 := makeCid("c")
-	k4 := makeCid("d")
-
-	if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustNotHave(markSet, k3)
-	mustNotHave(markSet, k4)
-
-	if err := markSet.BeginCriticalSection(); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustHave(markSet, k3)
-	mustHave(markSet, k4)
-
-	if err := markSet.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	markSet, err = env.Recover("test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	mustHave(markSet, k1)
-	mustHave(markSet, k2)
-	mustHave(markSet, k3)
-	mustHave(markSet, k4)
-
-	markSet.EndCriticalSection()
-
-	if err := markSet.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = env.Recover("test")
-	if err == nil {
-		t.Fatal("expected recovery to fail")
-	}
-}

blockstore/splitstore/splitstore.go

@@ -129,6 +129,8 @@ type SplitStore struct {
 
 	headChangeMx sync.Mutex
 
+	coldPurgeSize int
+
 	chain ChainAccessor
 	ds    dstore.Datastore
 	cold  bstore.Blockstore
@@ -156,10 +158,6 @@ type SplitStore struct {
 	txnRefsMx       sync.Mutex
 	txnRefs         map[cid.Cid]struct{}
 	txnMissing      map[cid.Cid]struct{}
-	txnMarkSet      MarkSet
-	txnSyncMx       sync.Mutex
-	txnSyncCond     sync.Cond
-	txnSync         bool
 
 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
@@ -196,10 +194,11 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		cold:       cold,
 		hot:        hots,
 		markSetEnv: markSetEnv,
+
+		coldPurgeSize: defaultColdPurgeSize,
 	}
 
 	ss.txnViewsCond.L = &ss.txnViewsMx
-	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())
 
 	if enableDebugLog {
@@ -209,14 +208,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		}
 	}
 
-	if ss.checkpointExists() {
-		log.Info("found compaction checkpoint; resuming compaction")
-		if err := ss.completeCompaction(); err != nil {
-			markSetEnv.Close() //nolint:errcheck
-			return nil, xerrors.Errorf("error resuming compaction: %w", err)
-		}
-	}
-
 	return ss, nil
 }
 
@@ -239,20 +230,6 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	// critical section
-	if s.txnMarkSet != nil {
-		has, err := s.txnMarkSet.Has(cid)
-		if err != nil {
-			return false, err
-		}
-
-		if has {
-			return s.has(cid)
-		}
-
-		return s.cold.Has(ctx, cid)
-	}
-
 	has, err := s.hot.Has(ctx, cid)
 
 	if err != nil {
@@ -280,20 +257,6 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	// critical section
-	if s.txnMarkSet != nil {
-		has, err := s.txnMarkSet.Has(cid)
-		if err != nil {
-			return nil, err
-		}
-
-		if has {
-			return s.get(cid)
-		}
-
-		return s.cold.Get(ctx, cid)
-	}
-
 	blk, err := s.hot.Get(ctx, cid)
 
 	switch err {
@@ -331,20 +294,6 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
-	// critical section
-	if s.txnMarkSet != nil {
-		has, err := s.txnMarkSet.Has(cid)
-		if err != nil {
-			return 0, err
-		}
-
-		if has {
-			return s.getSize(cid)
-		}
-
-		return s.cold.GetSize(ctx, cid)
-	}
-
 	size, err := s.hot.GetSize(ctx, cid)
 
 	switch err {
@@ -383,12 +332,6 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {
 
 	s.debug.LogWrite(blk)
 
-	// critical section
-	if s.txnMarkSet != nil {
-		s.markLiveRefs([]cid.Cid{blk.Cid()})
-		return nil
-	}
-
 	s.trackTxnRef(blk.Cid())
 	return nil
 }
@@ -434,12 +377,6 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {
 
 	s.debug.LogWriteMany(blks)
 
-	// critical section
-	if s.txnMarkSet != nil {
-		s.markLiveRefs(batch)
-		return nil
-	}
-
 	s.trackTxnRefMany(batch)
 	return nil
 }
@@ -499,23 +436,6 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		return cb(data)
 	}
 
-	// critical section
-	s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
-	if s.txnMarkSet != nil {
-		has, err := s.txnMarkSet.Has(cid)
-		s.txnLk.RUnlock()
-
-		if err != nil {
-			return err
-		}
-
-		if has {
-			return s.view(cid, cb)
-		}
-
-		return s.cold.View(ctx, cid, cb)
-	}
-
 	// views are (optimistically) protected two-fold:
 	// - if there is an active transaction, then the reference is protected.
 	// - if there is no active transaction, active views are tracked in a
@@ -665,11 +585,6 @@ func (s *SplitStore) Close() error {
 	}
 
 	if atomic.LoadInt32(&s.compacting) == 1 {
-		s.txnSyncMx.Lock()
-		s.txnSync = true
-		s.txnSyncCond.Broadcast()
-		s.txnSyncMx.Unlock()
-
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)

blockstore/splitstore/splitstore_check.go

@@ -89,7 +89,7 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	coldCnt := new(int64)
 	missingCnt := new(int64)
 
-	visitor, err := s.markSetEnv.New("check", 0)
+	visitor, err := s.markSetEnv.Create("check", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}

blockstore/splitstore/splitstore_compact.go

@@ -3,9 +3,8 @@ package splitstore
 import (
 	"bytes"
 	"errors"
-	"os"
-	"path/filepath"
 	"runtime"
+	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -49,10 +48,6 @@ var (
 	// SyncGapTime is the time delay from a tipset's min timestamp before we decide
 	// there is a sync gap
 	SyncGapTime = time.Minute
-
-	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
-	// we have synced.
-	SyncWaitTime = 30 * time.Second
 )
 
 var (
@@ -62,6 +57,8 @@ var (
 
 const (
 	batchSize = 16384
+
+	defaultColdPurgeSize = 7_000_000
 )
 
 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -144,9 +141,9 @@ func (s *SplitStore) isNearUpgrade(epoch abi.ChainEpoch) bool {
 // transactionally protect incoming tipsets
 func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
 
 	if !s.txnActive {
-		s.txnLk.RUnlock()
 		return
 	}
 
@@ -155,115 +152,12 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 		cids = append(cids, ts.Cids()...)
 	}
 
-	if len(cids) == 0 {
-		s.txnLk.RUnlock()
-		return
-	}
-
-	// critical section
-	if s.txnMarkSet != nil {
-		curTs := apply[len(apply)-1]
-		timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
-		doSync := time.Since(timestamp) < SyncWaitTime
-		go func() {
-			if doSync {
-				defer func() {
-					s.txnSyncMx.Lock()
-					defer s.txnSyncMx.Unlock()
-					s.txnSync = true
-					s.txnSyncCond.Broadcast()
-				}()
-			}
-			defer s.txnLk.RUnlock()
-			s.markLiveRefs(cids)
-
-		}()
-		return
-	}
-
 	s.trackTxnRefMany(cids)
-	s.txnLk.RUnlock()
-}
-
-func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
-	log.Debugf("marking %d live refs", len(cids))
-	startMark := time.Now()
-
-	count := new(int32)
-	visitor := newConcurrentVisitor()
-	walkObject := func(c cid.Cid) error {
-		return s.walkObjectIncomplete(c, visitor,
-			func(c cid.Cid) error {
-				if isUnitaryObject(c) {
-					return errStopWalk
-				}
-
-				visit, err := s.txnMarkSet.Visit(c)
-				if err != nil {
-					return xerrors.Errorf("error visiting object: %w", err)
-				}
-
-				if !visit {
-					return errStopWalk
-				}
-
-				atomic.AddInt32(count, 1)
-				return nil
-			},
-			func(missing cid.Cid) error {
-				log.Warnf("missing object reference %s in %s", missing, c)
-				return errStopWalk
-			})
-	}
-
-	// optimize the common case of single put
-	if len(cids) == 1 {
-		if err := walkObject(cids[0]); err != nil {
-			log.Errorf("error marking tipset refs: %s", err)
-		}
-		log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
-		return
-	}
-
-	workch := make(chan cid.Cid, len(cids))
-	for _, c := range cids {
-		workch <- c
-	}
-	close(workch)
-
-	worker := func() error {
-		for c := range workch {
-			if err := walkObject(c); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	workers := runtime.NumCPU() / 2
-	if workers < 2 {
-		workers = 2
-	}
-	if workers > len(cids) {
-		workers = len(cids)
-	}
-
-	g := new(errgroup.Group)
-	for i := 0; i < workers; i++ {
-		g.Go(worker)
-	}
-
-	if err := g.Wait(); err != nil {
-		log.Errorf("error marking tipset refs: %s", err)
-	}
-
-	log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
 }
 
 // transactionally protect a view
 func (s *SplitStore) protectView(c cid.Cid) {
-	//  the txnLk is held for read
+	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
 	if s.txnActive {
@@ -493,12 +387,6 @@ func (s *SplitStore) compact(curTs *types.TipSet) {
 }
 
 func (s *SplitStore) doCompact(curTs *types.TipSet) error {
-	if s.checkpointExists() {
-		// this really shouldn't happen, but if it somehow does, it means that the hotstore
-		// might be potentially inconsistent; abort compaction and notify the user to intervene.
-		return xerrors.Errorf("checkpoint exists; aborting compaction")
-	}
-
 	currentEpoch := curTs.Height()
 	boundaryEpoch := currentEpoch - CompactionBoundary
 
@@ -510,7 +398,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 
 	log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "inclMsgsEpoch", inclMsgsEpoch, "compactionIndex", s.compactionIndex)
 
-	markSet, err := s.markSetEnv.New("live", s.markSetSize)
+	markSet, err := s.markSetEnv.Create("live", s.markSetSize)
 	if err != nil {
 		return xerrors.Errorf("error creating mark set: %w", err)
 	}
@@ -521,6 +409,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}
 
+	// we are ready for concurrent marking
+	s.beginTxnMarking(markSet)
+
 	// 0. track all protected references at beginning of compaction; anything added later should
 	//    be transactionally protected by the write
 	log.Info("protecting references with registered protectors")
@@ -534,7 +425,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("marking reachable objects")
 	startMark := time.Now()
 
-	count := new(int64)
+	var count int64
 	err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
@@ -550,7 +441,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 				return errStopWalk
 			}
 
-			atomic.AddInt64(count, 1)
+			count++
 			return nil
 		})
 
@@ -558,9 +449,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error marking: %w", err)
 	}
 
-	s.markSetSize = *count + *count>>2 // overestimate a bit
+	s.markSetSize = count + count>>2 // overestimate a bit
 
-	log.Infow("marking done", "took", time.Since(startMark), "marked", *count)
+	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
 
 	if err := s.checkClosing(); err != nil {
 		return err
@@ -580,15 +471,10 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("collecting cold objects")
 	startCollect := time.Now()
 
-	coldw, err := NewColdSetWriter(s.coldSetPath())
-	if err != nil {
-		return xerrors.Errorf("error creating coldset: %w", err)
-	}
-	defer coldw.Close() //nolint:errcheck
-
 	// some stats for logging
 	var hotCnt, coldCnt int
 
+	cold := make([]cid.Cid, 0, s.coldPurgeSize)
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -602,9 +488,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		}
 
 		// it's cold, mark it as candidate for move
-		if err := coldw.Write(c); err != nil {
-			return xerrors.Errorf("error writing cid to coldstore: %w", err)
-		}
+		cold = append(cold, c)
 		coldCnt++
 
 		return nil
@@ -614,12 +498,12 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}
 
-	if err := coldw.Close(); err != nil {
-		return xerrors.Errorf("error closing coldset: %w", err)
-	}
-
 	log.Infow("cold collection done", "took", time.Since(startCollect))
 
+	if coldCnt > 0 {
+		s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
+	}
+
 	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
 	stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
 	stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
@@ -637,17 +521,11 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}
 
-	coldr, err := NewColdSetReader(s.coldSetPath())
-	if err != nil {
-		return xerrors.Errorf("error opening coldset: %w", err)
-	}
-	defer coldr.Close() //nolint:errcheck
-
 	// 3. copy the cold objects to the coldstore -- if we have one
 	if !s.cfg.DiscardColdBlocks {
 		log.Info("moving cold objects to the coldstore")
 		startMove := time.Now()
-		err = s.moveColdBlocks(coldr)
+		err = s.moveColdBlocks(cold)
 		if err != nil {
 			return xerrors.Errorf("error moving cold objects: %w", err)
 		}
@@ -656,64 +534,41 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
-
-		if err := coldr.Reset(); err != nil {
-			return xerrors.Errorf("error resetting coldset: %w", err)
-		}
 	}
 
-	// 4. Purge cold objects with checkpointing for recovery.
-	// This is the critical section of compaction, whereby any cold object not in the markSet is
-	// considered already deleted.
-	// We delete cold objects in batches, holding the transaction lock, where we check the markSet
-	// again for new references created by the VM.
-	// After each batch, we write a checkpoint to disk; if the process is interrupted before completion,
-	// the process will continue from the checkpoint in the next recovery.
-	if err := s.beginCriticalSection(markSet); err != nil {
-		return xerrors.Errorf("error beginning critical section: %w", err)
-	}
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	// wait for the head to catch up so that the current tipset is marked
-	s.waitForSync()
-
-	if err := s.checkClosing(); err != nil {
-		return err
-	}
-
-	checkpoint, err := NewCheckpoint(s.checkpointPath())
+	// 4. sort cold objects so that the dags with most references are deleted first
+	//    this ensures that we can't refer to a dag with its consituents already deleted, ie
+	//    we lave no dangling references.
+	log.Info("sorting cold objects")
+	startSort := time.Now()
+	err = s.sortObjects(cold)
 	if err != nil {
-		return xerrors.Errorf("error creating checkpoint: %w", err)
+		return xerrors.Errorf("error sorting objects: %w", err)
+	}
+	log.Infow("sorting done", "took", time.Since(startSort))
+
+	// 4.1 protect transactional refs once more
+	//     strictly speaking, this is not necessary as purge will do it before deleting each
+	//     batch.  however, there is likely a largish number of references accumulated during
+	//     ths sort and this protects before entering pruge context.
+	err = s.protectTxnRefs(markSet)
+	if err != nil {
+		return xerrors.Errorf("error protecting transactional refs: %w", err)
+	}
+
+	if err := s.checkClosing(); err != nil {
+		return err
 	}
-	defer checkpoint.Close() //nolint:errcheck
 
 	// 5. purge cold objects from the hotstore, taking protected references into account
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
-	err = s.purge(coldr, checkpoint, markSet)
+	err = s.purge(cold, markSet)
 	if err != nil {
-		return xerrors.Errorf("error purging cold objects: %w", err)
+		return xerrors.Errorf("error purging cold blocks: %w", err)
 	}
 	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
 
-	s.endCriticalSection()
-
-	if err := checkpoint.Close(); err != nil {
-		log.Warnf("error closing checkpoint: %s", err)
-	}
-	if err := os.Remove(s.checkpointPath()); err != nil {
-		log.Warnf("error removing checkpoint: %s", err)
-	}
-	if err := coldr.Close(); err != nil {
-		log.Warnf("error closing coldset: %s", err)
-	}
-	if err := os.Remove(s.coldSetPath()); err != nil {
-		log.Warnf("error removing coldset: %s", err)
-	}
-
 	// we are done; do some housekeeping
 	s.endTxnProtect()
 	s.gcHotstore()
@@ -744,51 +599,12 @@ func (s *SplitStore) beginTxnProtect() {
 	defer s.txnLk.Unlock()
 
 	s.txnActive = true
-	s.txnSync = false
 	s.txnRefs = make(map[cid.Cid]struct{})
 	s.txnMissing = make(map[cid.Cid]struct{})
 }
 
-func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
-	log.Info("beginning critical section")
-
-	// do that once first to get the bulk before the markset is in critical section
-	if err := s.protectTxnRefs(markSet); err != nil {
-		return xerrors.Errorf("error protecting transactional references: %w", err)
-	}
-
-	if err := markSet.BeginCriticalSection(); err != nil {
-		return xerrors.Errorf("error beginning critical section for markset: %w", err)
-	}
-
-	s.txnLk.Lock()
-	defer s.txnLk.Unlock()
-
-	s.txnMarkSet = markSet
-
-	// and do it again while holding the lock to mark references that might have been created
-	// in the meantime and avoid races of the type Has->txnRef->enterCS->Get fails because
-	// it's not in the markset
-	if err := s.protectTxnRefs(markSet); err != nil {
-		return xerrors.Errorf("error protecting transactional references: %w", err)
-	}
-
-	return nil
-}
-
-func (s *SplitStore) waitForSync() {
-	log.Info("waiting for sync")
-	startWait := time.Now()
-	defer func() {
-		log.Infow("waiting for sync done", "took", time.Since(startWait))
-	}()
-
-	s.txnSyncMx.Lock()
-	defer s.txnSyncMx.Unlock()
-
-	for !s.txnSync {
-		s.txnSyncCond.Wait()
-	}
+func (s *SplitStore) beginTxnMarking(markSet MarkSet) {
+	log.Info("beginning transactional marking")
 }
 
 func (s *SplitStore) endTxnProtect() {
@@ -800,20 +616,8 @@ func (s *SplitStore) endTxnProtect() {
 	}
 
 	s.txnActive = false
-	s.txnSync = false
 	s.txnRefs = nil
 	s.txnMissing = nil
-	s.txnMarkSet = nil
-}
-
-func (s *SplitStore) endCriticalSection() {
-	log.Info("ending critical section")
-
-	s.txnLk.Lock()
-	defer s.txnLk.Unlock()
-
-	s.txnMarkSet.EndCriticalSection()
-	s.txnMarkSet = nil
 }
 
 func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
@@ -1053,7 +857,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
 	return nil
 }
 
-// internal version used during compaction and related operations
+// internal version used by walk
 func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
@@ -1088,34 +892,10 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 	return s.cold.Has(s.ctx, c)
 }
 
-func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
-	blk, err := s.hot.Get(s.ctx, c)
-	switch err {
-	case nil:
-		return blk, nil
-	case bstore.ErrNotFound:
-		return s.cold.Get(s.ctx, c)
-	default:
-		return nil, err
-	}
-}
-
-func (s *SplitStore) getSize(c cid.Cid) (int, error) {
-	sz, err := s.hot.GetSize(s.ctx, c)
-	switch err {
-	case nil:
-		return sz, nil
-	case bstore.ErrNotFound:
-		return s.cold.GetSize(s.ctx, c)
-	default:
-		return 0, err
-	}
-}
-
-func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
+func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 	batch := make([]blocks.Block, 0, batchSize)
 
-	err := coldr.ForEach(func(c cid.Cid) error {
+	for _, c := range cold {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
@@ -1124,7 +904,7 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 		if err != nil {
 			if err == bstore.ErrNotFound {
 				log.Warnf("hotstore missing block %s", c)
-				return nil
+				continue
 			}
 
 			return xerrors.Errorf("error retrieving block %s from hotstore: %w", c, err)
@@ -1138,12 +918,6 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 			}
 			batch = batch[:0]
 		}
-
-		return nil
-	})
-
-	if err != nil {
-		return xerrors.Errorf("error iterating coldset: %w", err)
 	}
 
 	if len(batch) > 0 {
@@ -1156,202 +930,177 @@ func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 	return nil
 }
 
-func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet MarkSet) error {
-	batch := make([]cid.Cid, 0, batchSize)
-	deadCids := make([]cid.Cid, 0, batchSize)
-
-	var purgeCnt, liveCnt int
-	defer func() {
-		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
-	}()
-
-	deleteBatch := func() error {
-		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
-
-		purgeCnt += pc
-		liveCnt += lc
-		batch = batch[:0]
-
-		return err
+// sorts a slice of objects heaviest first -- it's a little expensive but worth the
+// guarantee that we don't leave dangling references behind, e.g. if we die in the middle
+// of a purge.
+func (s *SplitStore) sortObjects(cids []cid.Cid) error {
+	// we cache the keys to avoid making a gazillion of strings
+	keys := make(map[cid.Cid]string)
+	key := func(c cid.Cid) string {
+		s, ok := keys[c]
+		if !ok {
+			s = string(c.Hash())
+			keys[c] = s
+		}
+		return s
 	}
 
-	err := coldr.ForEach(func(c cid.Cid) error {
-		batch = append(batch, c)
-		if len(batch) == batchSize {
-			return deleteBatch()
+	// compute sorting weights as the cumulative number of DAG links
+	weights := make(map[string]int)
+	for _, c := range cids {
+		// this can take quite a while, so check for shutdown with every opportunity
+		if err := s.checkClosing(); err != nil {
+			return err
 		}
 
-		return nil
+		w := s.getObjectWeight(c, weights, key)
+		weights[key(c)] = w
+	}
+
+	// sort!
+	sort.Slice(cids, func(i, j int) bool {
+		wi := weights[key(cids[i])]
+		wj := weights[key(cids[j])]
+		if wi == wj {
+			return bytes.Compare(cids[i].Hash(), cids[j].Hash()) > 0
+		}
+
+		return wi > wj
 	})
 
-	if err != nil {
-		return err
-	}
-
-	if len(batch) > 0 {
-		return deleteBatch()
-	}
-
 	return nil
 }
 
-func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoint, markSet MarkSet) (purgeCnt int, liveCnt int, err error) {
-	if err := s.checkClosing(); err != nil {
-		return 0, 0, err
+func (s *SplitStore) getObjectWeight(c cid.Cid, weights map[string]int, key func(cid.Cid) string) int {
+	w, ok := weights[key(c)]
+	if ok {
+		return w
 	}
 
-	s.txnLk.Lock()
-	defer s.txnLk.Unlock()
+	// we treat block headers specially to avoid walking the entire chain
+	var hdr types.BlockHeader
+	err := s.view(c, func(data []byte) error {
+		return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
+	})
+	if err == nil {
+		w1 := s.getObjectWeight(hdr.ParentStateRoot, weights, key)
+		weights[key(hdr.ParentStateRoot)] = w1
 
-	for _, c := range batch {
-		has, err := markSet.Has(c)
-		if err != nil {
-			return 0, 0, xerrors.Errorf("error checking markset for liveness: %w", err)
-		}
+		w2 := s.getObjectWeight(hdr.Messages, weights, key)
+		weights[key(hdr.Messages)] = w2
 
-		if has {
-			liveCnt++
+		return 1 + w1 + w2
+	}
+
+	var links []cid.Cid
+	err = s.view(c, func(data []byte) error {
+		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
+			links = append(links, c)
+		})
+	})
+	if err != nil {
+		return 1
+	}
+
+	w = 1
+	for _, c := range links {
+		// these are internal refs, so dags will be dags
+		if c.Prefix().Codec != cid.DagCBOR {
+			w++
 			continue
 		}
 
-		deadCids = append(deadCids, c)
+		wc := s.getObjectWeight(c, weights, key)
+		weights[key(c)] = wc
+
+		w += wc
 	}
 
-	if len(deadCids) == 0 {
-		if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
-			return 0, 0, xerrors.Errorf("error setting checkpoint: %w", err)
+	return w
+}
+
+func (s *SplitStore) purgeBatch(cids []cid.Cid, deleteBatch func([]cid.Cid) error) error {
+	if len(cids) == 0 {
+		return nil
+	}
+
+	// we don't delete one giant batch of millions of objects, but rather do smaller batches
+	// so that we don't stop the world for an extended period of time
+	done := false
+	for i := 0; !done; i++ {
+		start := i * batchSize
+		end := start + batchSize
+		if end >= len(cids) {
+			end = len(cids)
+			done = true
 		}
 
-		return 0, liveCnt, nil
+		err := deleteBatch(cids[start:end])
+		if err != nil {
+			return xerrors.Errorf("error deleting batch: %w", err)
+		}
 	}
 
-	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
-		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
-	}
-
-	s.debug.LogDelete(deadCids)
-	purgeCnt = len(deadCids)
-
-	if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
-		return purgeCnt, liveCnt, xerrors.Errorf("error setting checkpoint: %w", err)
-	}
-
-	return purgeCnt, liveCnt, nil
-}
-
-func (s *SplitStore) coldSetPath() string {
-	return filepath.Join(s.path, "coldset")
-}
-
-func (s *SplitStore) checkpointPath() string {
-	return filepath.Join(s.path, "checkpoint")
-}
-
-func (s *SplitStore) checkpointExists() bool {
-	_, err := os.Stat(s.checkpointPath())
-	return err == nil
-}
-
-func (s *SplitStore) completeCompaction() error {
-	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
-	if err != nil {
-		return xerrors.Errorf("error opening checkpoint: %w", err)
-	}
-	defer checkpoint.Close() //nolint:errcheck
-
-	coldr, err := NewColdSetReader(s.coldSetPath())
-	if err != nil {
-		return xerrors.Errorf("error opening coldset: %w", err)
-	}
-	defer coldr.Close() //nolint:errcheck
-
-	markSet, err := s.markSetEnv.Recover("live")
-	if err != nil {
-		return xerrors.Errorf("error recovering markset: %w", err)
-	}
-	defer markSet.Close() //nolint:errcheck
-
-	// PURGE
-	log.Info("purging cold objects from the hotstore")
-	startPurge := time.Now()
-	err = s.completePurge(coldr, checkpoint, last, markSet)
-	if err != nil {
-		return xerrors.Errorf("error purging cold objects: %w", err)
-	}
-	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
-
-	markSet.EndCriticalSection()
-
-	if err := checkpoint.Close(); err != nil {
-		log.Warnf("error closing checkpoint: %s", err)
-	}
-	if err := os.Remove(s.checkpointPath()); err != nil {
-		log.Warnf("error removing checkpoint: %s", err)
-	}
-	if err := coldr.Close(); err != nil {
-		log.Warnf("error closing coldset: %s", err)
-	}
-	if err := os.Remove(s.coldSetPath()); err != nil {
-		log.Warnf("error removing coldset: %s", err)
-	}
-
-	// Note: at this point we can start the splitstore; a compaction should run on
-	//       the first head change, which will trigger gc on the hotstore.
-	//       We don't mind the second (back-to-back) compaction as the head will
-	//       have advanced during marking and coldset accumulation.
 	return nil
 }
 
-func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, start cid.Cid, markSet MarkSet) error {
-	if !start.Defined() {
-		return s.purge(coldr, checkpoint, markSet)
-	}
-
-	seeking := true
-	batch := make([]cid.Cid, 0, batchSize)
+func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSet) error {
 	deadCids := make([]cid.Cid, 0, batchSize)
-
 	var purgeCnt, liveCnt int
 	defer func() {
 		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
 	}()
 
-	deleteBatch := func() error {
-		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
+	return s.purgeBatch(cids,
+		func(cids []cid.Cid) error {
+			deadCids := deadCids[:0]
 
-		purgeCnt += pc
-		liveCnt += lc
-		batch = batch[:0]
+			for {
+				if err := s.checkClosing(); err != nil {
+					return err
+				}
 
-		return err
-	}
+				s.txnLk.Lock()
+				if len(s.txnRefs) == 0 {
+					// keep the lock!
+					break
+				}
 
-	err := coldr.ForEach(func(c cid.Cid) error {
-		if seeking {
-			if start.Equals(c) {
-				seeking = false
+				// unlock and protect
+				s.txnLk.Unlock()
+
+				err := s.protectTxnRefs(markSet)
+				if err != nil {
+					return xerrors.Errorf("error protecting transactional refs: %w", err)
+				}
 			}
 
+			defer s.txnLk.Unlock()
+
+			for _, c := range cids {
+				live, err := markSet.Has(c)
+				if err != nil {
+					return xerrors.Errorf("error checking for liveness: %w", err)
+				}
+
+				if live {
+					liveCnt++
+					continue
+				}
+
+				deadCids = append(deadCids, c)
+			}
+
+			err := s.hot.DeleteMany(s.ctx, deadCids)
+			if err != nil {
+				return xerrors.Errorf("error purging cold objects: %w", err)
+			}
+
+			s.debug.LogDelete(deadCids)
+
+			purgeCnt += len(deadCids)
 			return nil
-		}
-
-		batch = append(batch, c)
-		if len(batch) == batchSize {
-			return deleteBatch()
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	if len(batch) > 0 {
-		return deleteBatch()
-	}
-
-	return nil
+		})
 }
 
 // I really don't like having this code, but we seem to have some occasional DAG references with

blockstore/splitstore/splitstore_test.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -22,14 +20,12 @@ import (
 	datastore "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
-	mh "github.com/multiformats/go-multihash"
 )
 
 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
 	WarmupBoundary = 0
-	SyncWaitTime = time.Millisecond
 	logging.SetLogLevel("splitstore", "DEBUG")
 }
 
@@ -84,17 +80,8 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal(err)
 	}
 
-	path, err := ioutil.TempDir("", "splitstore.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
-
 	// open the splitstore
-	ss, err := Open(path, ds, hot, cold, cfg)
+	ss, err := Open("", ds, hot, cold, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -138,10 +125,6 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}
 
 	waitForCompaction := func() {
-		ss.txnSyncMx.Lock()
-		ss.txnSync = true
-		ss.txnSyncCond.Broadcast()
-		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -276,17 +259,8 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	path, err := ioutil.TempDir("", "splitstore.*")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() {
-		_ = os.RemoveAll(path)
-	})
-
 	// open the splitstore
-	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	ss, err := Open("", ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -331,10 +305,6 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 
 	waitForCompaction := func() {
-		ss.txnSyncMx.Lock()
-		ss.txnSync = true
-		ss.txnSyncCond.Broadcast()
-		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -456,25 +426,17 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app
 
 type mockStore struct {
 	mx  sync.Mutex
-	set map[string]blocks.Block
+	set map[cid.Cid]blocks.Block
 }
 
 func newMockStore() *mockStore {
-	return &mockStore{set: make(map[string]blocks.Block)}
-}
-
-func (b *mockStore) keyOf(c cid.Cid) string {
-	return string(c.Hash())
-}
-
-func (b *mockStore) cidOf(k string) cid.Cid {
-	return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
+	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
 }
 
 func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
-	_, ok := b.set[b.keyOf(cid)]
+	_, ok := b.set[cid]
 	return ok, nil
 }
 
@@ -484,7 +446,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	blk, ok := b.set[b.keyOf(cid)]
+	blk, ok := b.set[cid]
 	if !ok {
 		return nil, blockstore.ErrNotFound
 	}
@@ -512,7 +474,7 @@ func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	b.set[b.keyOf(blk.Cid())] = blk
+	b.set[blk.Cid()] = blk
 	return nil
 }
 
@@ -521,7 +483,7 @@ func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
 	defer b.mx.Unlock()
 
 	for _, blk := range blks {
-		b.set[b.keyOf(blk.Cid())] = blk
+		b.set[blk.Cid()] = blk
 	}
 	return nil
 }
@@ -530,7 +492,7 @@ func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()
 
-	delete(b.set, b.keyOf(cid))
+	delete(b.set, cid)
 	return nil
 }
 
@@ -539,7 +501,7 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	defer b.mx.Unlock()
 
 	for _, c := range cids {
-		delete(b.set, b.keyOf(c))
+		delete(b.set, c)
 	}
 	return nil
 }
@@ -553,7 +515,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
 	defer b.mx.Unlock()
 
 	for c := range b.set {
-		err := f(b.cidOf(c))
+		err := f(c)
 		if err != nil {
 			return err
 		}

blockstore/splitstore/splitstore_warmup.go

@@ -62,7 +62,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	xcount := new(int64)
 	missing := new(int64)
 
-	visitor, err := s.markSetEnv.New("warmup", 0)
+	visitor, err := s.markSetEnv.Create("warmup", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}

build/bootstrap/butterflynet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBdRCBLUeKvoy22u5DcXs61adFn31v8WWCZgmBjDCjbsC
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDUQJBA18njjXnG9RtLxoN3muvdU7PEy55QorUEsdAqdy

build/params_butterfly.go

@@ -42,7 +42,8 @@ const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
 const UpgradeChocolateHeight = -17
 
-const UpgradeOhSnapHeight = 240
+// 2022-01-17T19:00:00Z
+const UpgradeOhSnapHeight = 30262
 
 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))

build/params_calibnet.go

@@ -54,8 +54,7 @@ const UpgradeHyperdriveHeight = 420
 
 const UpgradeChocolateHeight = 312746
 
-// 2022-02-10T19:23:00Z
-const UpgradeOhSnapHeight = 682006
+const UpgradeOhSnapHeight = 99999999
 
 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))

build/params_shared_funcs.go

@@ -13,6 +13,17 @@ import (
 
 func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
 func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func IndexerIngestTopic(netName dtypes.NetworkName) string {
+
+	nn := string(netName)
+	// The network name testnetnet is here for historical reasons.
+	// Going forward we aim to use the name `mainnet` where possible.
+	if nn == "testnetnet" {
+		nn = "mainnet"
+	}
+
+	return "/indexer/ingest/" + nn
+}
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }

build/proof-params/parameters.json

@@ -1,54 +1,4 @@
 {
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.params": {
-    "cid": "Qma5WL6abSqYg9uUQAZ3EHS286bsNsha7oAGsJBD48Bq2q",
-    "digest": "c3ad7bb549470b82ad52ed070aebb4f4",
-    "sector_size": 536870912
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-61fa69f38b9cc771ba27b670124714b4ea77fbeae05e377fb859c4a43b73a30c.vk": {
-    "cid": "QmUa7f9JtJMsqJJ3s3ZXk6WyF4xJLE8FiqYskZGgk8GCDv",
-    "digest": "994c5b7d450ca9da348c910689f2dc7f",
-    "sector_size": 536870912
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.params": {
-    "cid": "QmQiT4qBGodrVNEgVTDXxBNDdPbaD8Ag7Sx3ZTq1zHX79S",
-    "digest": "5aedd2cf3e5c0a15623d56a1b43110ad",
-    "sector_size": 8388608
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-92180959e1918d26350b8e6cfe217bbdd0a2d8de51ebec269078b364b715ad63.vk": {
-    "cid": "QmdcpKUQvHM8RFRVKbk1yHfEqMcBzhtFWKRp9SNEmWq37i",
-    "digest": "abd80269054d391a734febdac0d2e687",
-    "sector_size": 8388608
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.params": {
-    "cid": "QmYM6Hg7mjmvA3ZHTsqkss1fkdyDju5dDmLiBZGJ5pz9y9",
-    "digest": "311f92a3e75036ced01b1c0025f1fa0c",
-    "sector_size": 2048
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-0-0-fb9e095bebdd77511c0269b967b4d87ba8b8a525edaa0e165de23ba454510194.vk": {
-    "cid": "QmaQsTLL3nc5dw6wAvaioJSBfd1jhQrA2o6ucFf7XeV74P",
-    "digest": "eadad9784969890d30f2749708c79771",
-    "sector_size": 2048
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.params": {
-    "cid": "QmNPc75iEfcahCwNKdqnWLtxnjspUGGR4iscjiz3wP3RtS",
-    "digest": "1b3cfd761a961543f9eb273e435a06a2",
-    "sector_size": 34359738368
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.vk": {
-    "cid": "QmdFFUe1gcz9MMHc6YW8aoV48w4ckvcERjt7PkydQAMfCN",
-    "digest": "3a6941983754737fde880d29c7094905",
-    "sector_size": 34359738368
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.params": {
-    "cid": "QmUB6xTVjzBQGuDNeyJMrrJ1byk58vhPm8eY2Lv9pgwanp",
-    "digest": "1a392e7b759fb18e036c7559b5ece816",
-    "sector_size": 68719476736
-  },
-  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.vk": {
-    "cid": "Qmd794Jty7k26XJ8Eg4NDEks65Qk8G4GVfGkwqvymv8HAg",
-    "digest": "80e366df2f1011953c2d01c7b7c9ee8e",
-    "sector_size": 68719476736
-  },
   "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
     "cid": "QmVxjFRyhmyQaZEtCh7nk2abc7LhFkzhnRX4rcHqCCpikR",
     "digest": "7610b9f82bfc88405b7a832b651ce2f6",

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.15.0-rc1"
+const BuildVersion = "1.15.0-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/beacon/drand/drand.go

@@ -142,7 +142,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 
 	go func() {
 		start := build.Clock.Now()
-		log.Debugw("start fetching randomness", "round", round)
+		log.Infow("start fetching randomness", "round", round)
 		resp, err := db.client.Get(ctx, round)
 
 		var br beacon.Response
@@ -152,7 +152,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 			br.Entry.Round = resp.Round()
 			br.Entry.Data = resp.Signature()
 		}
-		log.Debugw("done fetching randomness", "round", round, "took", build.Clock.Since(start))
+		log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
 		out <- br
 		close(out)
 	}()

chain/sub/incoming.go

@@ -1,19 +1,24 @@
 package sub
 
 import (
+	"bytes"
 	"context"
-	"fmt"
+	"encoding/binary"
+	"sync"
 	"time"
 
 	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/sub/ratelimit"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/impl/client"
+	"github.com/filecoin-project/lotus/node/impl/full"
 	lru "github.com/hashicorp/golang-lru"
 	blocks "github.com/ipfs/go-block-format"
 	bserv "github.com/ipfs/go-blockservice"
@@ -168,12 +173,12 @@ func fetchCids(
 	cidIndex := make(map[cid.Cid]int)
 	for i, c := range cids {
 		if c.Prefix() != msgCidPrefix {
-			return fmt.Errorf("invalid msg CID: %s", c)
+			return xerrors.Errorf("invalid msg CID: %s", c)
 		}
 		cidIndex[c] = i
 	}
 	if len(cids) != len(cidIndex) {
-		return fmt.Errorf("duplicate CIDs in fetchCids input")
+		return xerrors.Errorf("duplicate CIDs in fetchCids input")
 	}
 
 	for block := range bserv.GetBlocks(ctx, cids) {
@@ -196,7 +201,7 @@ func fetchCids(
 	if len(cidIndex) > 0 {
 		err := ctx.Err()
 		if err == nil {
-			err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
+			err = xerrors.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
 		}
 		return err
 	}
@@ -444,3 +449,168 @@ func recordFailure(ctx context.Context, metric *stats.Int64Measure, failureType
 	)
 	stats.Record(ctx, metric.M(1))
 }
+
+type peerMsgInfo struct {
+	peerID    peer.ID
+	lastCid   cid.Cid
+	lastSeqno uint64
+	rateLimit *ratelimit.Window
+	mutex     sync.Mutex
+}
+
+type IndexerMessageValidator struct {
+	self peer.ID
+
+	peerCache *lru.TwoQueueCache
+	chainApi  full.ChainModuleAPI
+	stateApi  full.StateModuleAPI
+}
+
+func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator {
+	peerCache, _ := lru.New2Q(8192)
+
+	return &IndexerMessageValidator{
+		self:      self,
+		peerCache: peerCache,
+		chainApi:  chainApi,
+		stateApi:  stateApi,
+	}
+}
+
+func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
+	// This chain-node should not be publishing its own messages.  These are
+	// relayed from market-nodes.  If a node appears to be local, reject it.
+	if pid == v.self {
+		log.Debug("ignoring indexer message from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+	originPeer := msg.GetFrom()
+	if originPeer == v.self {
+		log.Debug("ignoring indexer message originating from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+
+	idxrMsg := dtsync.Message{}
+	err := idxrMsg.UnmarshalCBOR(bytes.NewBuffer(msg.Data))
+	if err != nil {
+		log.Errorw("Could not decode indexer pubsub message", "err", err)
+		return pubsub.ValidationReject
+	}
+	if len(idxrMsg.ExtraData) == 0 {
+		log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
+		return pubsub.ValidationIgnore
+	}
+
+	// Get miner info from lotus
+	minerAddr, err := address.NewFromBytes(idxrMsg.ExtraData)
+	if err != nil {
+		log.Warnw("cannot parse extra data as miner address", "err", err, "extraData", idxrMsg.ExtraData)
+		return pubsub.ValidationReject
+	}
+
+	minerID := minerAddr.String()
+	msgCid := idxrMsg.Cid
+
+	var msgInfo *peerMsgInfo
+	val, ok := v.peerCache.Get(minerID)
+	if !ok {
+		msgInfo = &peerMsgInfo{}
+	} else {
+		msgInfo = val.(*peerMsgInfo)
+	}
+
+	// Lock this peer's message info.
+	msgInfo.mutex.Lock()
+	defer msgInfo.mutex.Unlock()
+
+	if ok {
+		// Reject replayed messages.
+		seqno := binary.BigEndian.Uint64(msg.Message.GetSeqno())
+		if seqno <= msgInfo.lastSeqno {
+			log.Debugf("ignoring replayed indexer message")
+			return pubsub.ValidationIgnore
+		}
+		msgInfo.lastSeqno = seqno
+	}
+
+	if !ok || originPeer != msgInfo.peerID {
+		// Check that the miner ID maps to the peer that sent the message.
+		err = v.authenticateMessage(ctx, minerAddr, originPeer)
+		if err != nil {
+			log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID)
+			stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+			return pubsub.ValidationReject
+		}
+		msgInfo.peerID = originPeer
+		if !ok {
+			// Add msgInfo to cache only after being authenticated.  If two
+			// messages from the same peer are handled concurrently, there is a
+			// small chance that one msgInfo could replace the other here when
+			// the info is first cached.  This is OK, so no need to prevent it.
+			v.peerCache.Add(minerID, msgInfo)
+		}
+	}
+
+	// See if message needs to be ignored due to rate limiting.
+	if v.rateLimitPeer(msgInfo, msgCid) {
+		return pubsub.ValidationIgnore
+	}
+
+	stats.Record(ctx, metrics.IndexerMessageValidationSuccess.M(1))
+	return pubsub.ValidationAccept
+}
+
+func (v *IndexerMessageValidator) rateLimitPeer(msgInfo *peerMsgInfo, msgCid cid.Cid) bool {
+	const (
+		msgLimit        = 5
+		msgTimeLimit    = 10 * time.Second
+		repeatTimeLimit = 2 * time.Hour
+	)
+
+	timeWindow := msgInfo.rateLimit
+
+	// Check overall message rate.
+	if timeWindow == nil {
+		timeWindow = ratelimit.NewWindow(msgLimit, msgTimeLimit)
+		msgInfo.rateLimit = timeWindow
+	} else if msgInfo.lastCid == msgCid {
+		// Check if this is a repeat of the previous message data.
+		if time.Since(timeWindow.Newest()) < repeatTimeLimit {
+			log.Warnw("ignoring repeated indexer message", "sender", msgInfo.peerID)
+			return true
+		}
+	}
+
+	err := timeWindow.Add()
+	if err != nil {
+		log.Warnw("ignoring indexer message", "sender", msgInfo.peerID, "err", err)
+		return true
+	}
+
+	msgInfo.lastCid = msgCid
+
+	return false
+}
+
+func (v *IndexerMessageValidator) authenticateMessage(ctx context.Context, minerAddress address.Address, peerID peer.ID) error {
+	ts, err := v.chainApi.ChainHead(ctx)
+	if err != nil {
+		return err
+	}
+
+	minerInfo, err := v.stateApi.StateMinerInfo(ctx, minerAddress, ts.Key())
+	if err != nil {
+		return err
+	}
+
+	if minerInfo.PeerId == nil {
+		return xerrors.New("no peer id for miner")
+	}
+	if *minerInfo.PeerId != peerID {
+		return xerrors.New("miner id does not map to peer that sent message")
+	}
+
+	return nil
+}

chain/sub/incoming_test.go

@@ -2,13 +2,20 @@
 package sub
 
 import (
+	"bytes"
 	"context"
 	"testing"
 
-	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
+	"github.com/filecoin-project/lotus/api/mocks"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/golang/mock/gomock"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
 type getter struct {
@@ -63,3 +70,65 @@ func TestFetchCidsWithDedup(t *testing.T) {
 		t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1])
 	}
 }
+
+func TestIndexerMessageValidator_Validate(t *testing.T) {
+	validCid, err := cid.Decode("QmbpDgg5kRLDgMxS8vPKNFXEcA6D5MC4CkuUdSWDVtHPGK")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name           string
+		selfPID        string
+		senderPID      string
+		extraData      []byte
+		wantValidation pubsub.ValidationResult
+	}{
+		{
+			name:           "invalid extra data is rejected",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ",
+			extraData:      []byte("f0127896"), // note, casting encoded address to byte is invalid.
+			wantValidation: pubsub.ValidationReject,
+		},
+		{
+			name:           "same sender and receiver is ignored",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			wantValidation: pubsub.ValidationIgnore,
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			mc := gomock.NewController(t)
+			node := mocks.NewMockFullNode(mc)
+			subject := NewIndexerMessageValidator(peer.ID(tc.selfPID), node, node)
+			message := dtsync.Message{
+				Cid:       validCid,
+				Addrs:     nil,
+				ExtraData: tc.extraData,
+			}
+			buf := bytes.NewBuffer(nil)
+			if err := message.MarshalCBOR(buf); err != nil {
+				t.Fatal(err)
+			}
+
+			topic := "topic"
+			pbm := &pb.Message{
+				Data:  buf.Bytes(),
+				Topic: &topic,
+				From:  nil,
+				Seqno: nil,
+			}
+			validate := subject.Validate(context.Background(), peer.ID(tc.senderPID), &pubsub.Message{
+				Message:       pbm,
+				ReceivedFrom:  peer.ID(tc.senderPID),
+				ValidatorData: nil,
+			})
+
+			if validate != tc.wantValidation {
+				t.Fatalf("expected %v but got %v", tc.wantValidation, validate)
+			}
+		})
+	}
+}

chain/sub/ratelimit/queue.go

@@ -0,0 +1,89 @@
+package ratelimit
+
+import "errors"
+
+var ErrRateLimitExceeded = errors.New("rate limit exceeded")
+
+type queue struct {
+	buf   []int64
+	count int
+	head  int
+	tail  int
+}
+
+// cap returns the queue capacity
+func (q *queue) cap() int {
+	return len(q.buf)
+}
+
+// len returns the number of items in the queue
+func (q *queue) len() int {
+	return q.count
+}
+
+// push adds an element to the end of the queue.
+func (q *queue) push(elem int64) error {
+	if q.count == len(q.buf) {
+		return ErrRateLimitExceeded
+	}
+
+	q.buf[q.tail] = elem
+	// Calculate new tail position.
+	q.tail = q.next(q.tail)
+	q.count++
+	return nil
+}
+
+// pop removes and returns the element from the front of the queue.
+func (q *queue) pop() int64 {
+	if q.count <= 0 {
+		panic("pop from empty queue")
+	}
+	ret := q.buf[q.head]
+
+	// Calculate new head position.
+	q.head = q.next(q.head)
+	q.count--
+
+	return ret
+}
+
+// front returns the element at the front of the queue.  This is the element
+// that would be returned by pop().  This call panics if the queue is empty.
+func (q *queue) front() int64 {
+	if q.count <= 0 {
+		panic("front() called when empty")
+	}
+	return q.buf[q.head]
+}
+
+// back returns the element at the back of the queue.  This call panics if the
+// queue is empty.
+func (q *queue) back() int64 {
+	if q.count <= 0 {
+		panic("back() called when empty")
+	}
+	return q.buf[q.prev(q.tail)]
+}
+
+// prev returns the previous buffer position wrapping around buffer.
+func (q *queue) prev(i int) int {
+	if i == 0 {
+		return len(q.buf) - 1
+	}
+	return (i - 1) % len(q.buf)
+}
+
+// next returns the next buffer position wrapping around buffer.
+func (q *queue) next(i int) int {
+	return (i + 1) % len(q.buf)
+}
+
+// truncate pops values that are less than or equal the specified threshold.
+func (q *queue) truncate(threshold int64) {
+	for q.count != 0 && q.buf[q.head] <= threshold {
+		// pop() without returning a value
+		q.head = q.next(q.head)
+		q.count--
+	}
+}

chain/sub/ratelimit/window.go

@@ -0,0 +1,70 @@
+package ratelimit
+
+import "time"
+
+// Window is a time windows for counting events within a span of time.  The
+// windows slides forward in time so that it spans from the most recent event
+// to size time in the past.
+type Window struct {
+	q    *queue
+	size int64
+}
+
+// NewWindow creates a new Window that limits the number of events to maximum
+// count of events within a duration of time.  The capacity sets the maximum
+// number of events, and size sets the span of time over which the events are
+// counted.
+func NewWindow(capacity int, size time.Duration) *Window {
+	return &Window{
+		q: &queue{
+			buf: make([]int64, capacity),
+		},
+		size: int64(size),
+	}
+}
+
+// Add attempts to append a new timestamp into the current window.  Previously
+// added values that are not not within `size` difference from the value being
+// added are first removed.  Add fails if adding the value would cause the
+// window to exceed capacity.
+func (w *Window) Add() error {
+	now := time.Now().UnixNano()
+	if w.Len() != 0 {
+		w.q.truncate(now - w.size)
+	}
+	return w.q.push(now)
+}
+
+// Cap returns the maximum number of items the window can hold.
+func (w *Window) Cap() int {
+	return w.q.cap()
+}
+
+// Len returns the number of elements currently in the window.
+func (w *Window) Len() int {
+	return w.q.len()
+}
+
+// Span returns the distance from the first to the last item in the window.
+func (w *Window) Span() time.Duration {
+	if w.q.len() < 2 {
+		return 0
+	}
+	return time.Duration(w.q.back() - w.q.front())
+}
+
+// Oldest returns the oldest timestamp in the window.
+func (w *Window) Oldest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.front())
+}
+
+// Newest returns the newest timestamp in the window.
+func (w *Window) Newest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.back())
+}

chain/sub/ratelimit/window_test.go

@@ -0,0 +1,61 @@
+package ratelimit
+
+import (
+	"testing"
+	"time"
+)
+
+func TestWindow(t *testing.T) {
+	const (
+		maxEvents = 3
+		timeLimit = 100 * time.Millisecond
+	)
+	w := NewWindow(maxEvents, timeLimit)
+	if w.Len() != 0 {
+		t.Fatal("q.Len() =", w.Len(), "expect 0")
+	}
+	if w.Cap() != maxEvents {
+		t.Fatal("q.Cap() =", w.Cap(), "expect 3")
+	}
+	if !w.Newest().IsZero() {
+		t.Fatal("expected newest to be zero time with empty window")
+	}
+	if !w.Oldest().IsZero() {
+		t.Fatal("expected oldest to be zero time with empty window")
+	}
+	if w.Span() != 0 {
+		t.Fatal("expected span to be zero time with empty window")
+	}
+
+	var err error
+	for i := 0; i < maxEvents; i++ {
+		err = w.Add()
+		if err != nil {
+			t.Fatalf("cannot add event %d", i)
+		}
+	}
+	if w.Len() != maxEvents {
+		t.Fatalf("q.Len() is %d, expected %d", w.Len(), maxEvents)
+	}
+	if err = w.Add(); err != ErrRateLimitExceeded {
+		t.Fatalf("add event %d within time limit should have failed with err: %s", maxEvents+1, ErrRateLimitExceeded)
+	}
+
+	time.Sleep(timeLimit)
+	if err = w.Add(); err != nil {
+		t.Fatalf("cannot add event after time limit: %s", err)
+	}
+
+	prev := w.Newest()
+	time.Sleep(timeLimit)
+	err = w.Add()
+	if err != nil {
+		t.Fatalf("cannot add event")
+	}
+	if w.Newest().Before(prev) {
+		t.Fatal("newest is before previous value")
+	}
+	if w.Oldest().Before(prev) {
+		t.Fatal("oldest is before previous value")
+	}
+}

cli/net.go

@@ -38,6 +38,9 @@ var NetCmd = &cli.Command{
 		NetBlockCmd,
 		NetStatCmd,
 		NetLimitCmd,
+		NetProtectAdd,
+		NetProtectRemove,
+		NetProtectList,
 	},
 }
 
@@ -708,3 +711,108 @@ var NetLimitCmd = &cli.Command{
 		return enc.Encode(result)
 	},
 }
+
+var NetProtectAdd = &cli.Command{
+	Name:      "protect",
+	Usage:     "Add one or more peer IDs to the list of protected peer connections",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectAdd(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("added to protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+var NetProtectRemove = &cli.Command{
+	Name:      "unprotect",
+	Usage:     "Remove one or more peer IDs from the list of protected peer connections.",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectRemove(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("removed from protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+// decodePeerIDsFromArgs decodes all the arguments present in cli.Context.Args as peer.ID.
+//
+// This function requires at least one argument to be present, and arguments must not be empty
+// string. Otherwise, an error is returned.
+func decodePeerIDsFromArgs(cctx *cli.Context) ([]peer.ID, error) {
+	pidArgs := cctx.Args().Slice()
+	if len(pidArgs) == 0 {
+		return nil, xerrors.Errorf("must specify at least one peer ID as an argument")
+	}
+	var pids []peer.ID
+	for _, pidStr := range pidArgs {
+		if pidStr == "" {
+			return nil, xerrors.Errorf("peer ID must not be empty")
+		}
+		pid, err := peer.Decode(pidStr)
+		if err != nil {
+			return nil, err
+		}
+		pids = append(pids, pid)
+	}
+	return pids, nil
+}
+
+var NetProtectList = &cli.Command{
+	Name:  "list-protected",
+	Usage: "List the peer IDs with protected connection.",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		pids, err := api.NetProtectList(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, pid := range pids {
+			fmt.Printf("%s\n", pid)
+		}
+		return nil
+	},
+}

cmd/lotus-miner/dagstore.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/dustin/go-humanize"
 	"github.com/fatih/color"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/lotus/api"
@@ -21,6 +23,8 @@ var dagstoreCmd = &cli.Command{
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
 		dagstoreGcCmd,
+		dagstorePieceIndexSizeCmd,
+		dagstoreLookupPiecesCmd,
 	},
 }
 
@@ -52,38 +56,7 @@ var dagstoreListShardsCmd = &cli.Command{
 			return err
 		}
 
-		if len(shards) == 0 {
-			return nil
-		}
-
-		tw := tablewriter.New(
-			tablewriter.Col("Key"),
-			tablewriter.Col("State"),
-			tablewriter.Col("Error"),
-		)
-
-		colors := map[string]color.Attribute{
-			"ShardStateAvailable": color.FgGreen,
-			"ShardStateServing":   color.FgBlue,
-			"ShardStateErrored":   color.FgRed,
-			"ShardStateNew":       color.FgYellow,
-		}
-
-		for _, s := range shards {
-			m := map[string]interface{}{
-				"Key": s.Key,
-				"State": func() string {
-					if c, ok := colors[s.State]; ok {
-						return color.New(c).Sprint(s.State)
-					}
-					return s.State
-				}(),
-				"Error": s.Error,
-			}
-			tw.Write(m)
-		}
-
-		return tw.Flush(os.Stdout)
+		return printTableShards(shards)
 	},
 }
 
@@ -265,3 +238,114 @@ var dagstoreGcCmd = &cli.Command{
 		return nil
 	},
 }
+
+func printTableShards(shards []api.DagstoreShardInfo) error {
+	if len(shards) == 0 {
+		return nil
+	}
+
+	tw := tablewriter.New(
+		tablewriter.Col("Key"),
+		tablewriter.Col("State"),
+		tablewriter.Col("Error"),
+	)
+
+	colors := map[string]color.Attribute{
+		"ShardStateAvailable": color.FgGreen,
+		"ShardStateServing":   color.FgBlue,
+		"ShardStateErrored":   color.FgRed,
+		"ShardStateNew":       color.FgYellow,
+	}
+
+	for _, s := range shards {
+		m := map[string]interface{}{
+			"Key": s.Key,
+			"State": func() string {
+				if c, ok := colors[s.State]; ok {
+					return color.New(c).Sprint(s.State)
+				}
+				return s.State
+			}(),
+			"Error": s.Error,
+		}
+		tw.Write(m)
+	}
+	return tw.Flush(os.Stdout)
+}
+
+var dagstorePieceIndexSizeCmd = &cli.Command{
+	Name:  "piece-index-size",
+	Usage: "Inspect the dagstore piece index size",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		size, err := marketsApi.DagstorePieceIndexSize(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(humanize.Bytes(uint64(size)))
+
+		return nil
+	},
+}
+
+var dagstoreLookupPiecesCmd = &cli.Command{
+	Name:      "lookup-pieces",
+	Usage:     "Lookup pieces that a given CID belongs to",
+	ArgsUsage: "<cid>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide a CID")
+		}
+
+		cidStr := cctx.Args().First()
+		cid, err := cid.Parse(cidStr)
+		if err != nil {
+			return fmt.Errorf("invalid CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shards, err := marketsApi.DagstoreLookupPieces(ctx, cid)
+		if err != nil {
+			return err
+		}
+
+		return printTableShards(shards)
+	},
+}

cmd/lotus-miner/index_provider.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/fatih/color"
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var indexProvCmd = &cli.Command{
+	Name:  "index",
+	Usage: "Manage the index provider on the markets subsystem",
+	Subcommands: []*cli.Command{
+		indexProvAnnounceCmd,
+		indexProvAnnounceAllCmd,
+	},
+}
+
+var indexProvAnnounceCmd = &cli.Command{
+	Name:      "announce",
+	ArgsUsage: "<deal proposal cid>",
+	Usage:     "Announce a deal to indexers so they can download its index",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide the deal proposal CID")
+		}
+
+		proposalCidStr := cctx.Args().First()
+		proposalCid, err := cid.Parse(proposalCidStr)
+		if err != nil {
+			return fmt.Errorf("invalid deal proposal CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return marketsApi.IndexerAnnounceDeal(ctx, proposalCid)
+	},
+}
+
+var indexProvAnnounceAllCmd = &cli.Command{
+	Name:  "announce-all",
+	Usage: "Announce all active deals to indexers so they can download its indices",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return marketsApi.IndexerAnnounceAllDeals(ctx)
+	},
+}

cmd/lotus-miner/info.go

@@ -466,7 +466,6 @@ var stateOrder = map[sealing.SectorState]stateMeta{}
 var stateList = []stateMeta{
 	{col: 39, state: "Total"},
 	{col: color.FgGreen, state: sealing.Proving},
-	{col: color.FgGreen, state: sealing.UpdateActivating},
 
 	{col: color.FgBlue, state: sealing.Empty},
 	{col: color.FgBlue, state: sealing.WaitDeals},
@@ -497,7 +496,6 @@ var stateList = []stateMeta{
 	{col: color.FgYellow, state: sealing.SubmitReplicaUpdate},
 	{col: color.FgYellow, state: sealing.ReplicaUpdateWait},
 	{col: color.FgYellow, state: sealing.FinalizeReplicaUpdate},
-	{col: color.FgYellow, state: sealing.ReleaseSectorKey},
 
 	{col: color.FgCyan, state: sealing.Terminating},
 	{col: color.FgCyan, state: sealing.TerminateWait},
@@ -526,7 +524,6 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.SnapDealsAddPieceFailed},
 	{col: color.FgRed, state: sealing.SnapDealsDealsExpired},
 	{col: color.FgRed, state: sealing.ReplicaUpdateFailed},
-	{col: color.FgRed, state: sealing.ReleaseSectorKeyFailed},
 }
 
 func init() {

cmd/lotus-miner/main.go

@@ -48,6 +48,7 @@ func main() {
 		lcli.WithCategory("market", retrievalDealsCmd),
 		lcli.WithCategory("market", dataTransfersCmd),
 		lcli.WithCategory("market", dagstoreCmd),
+		lcli.WithCategory("market", indexProvCmd),
 		lcli.WithCategory("storage", sectorsCmd),
 		lcli.WithCategory("storage", provingCmd),
 		lcli.WithCategory("storage", storageCmd),

cmd/lotus-miner/sectors.go

@@ -55,7 +55,6 @@ var sectorsCmd = &cli.Command{
 		sectorsTerminateCmd,
 		sectorsRemoveCmd,
 		sectorsSnapUpCmd,
-		sectorsSnapAbortCmd,
 		sectorsMarkForUpgradeCmd,
 		sectorsStartSealCmd,
 		sectorsSealDelayCmd,
@@ -1521,31 +1520,6 @@ var sectorsSnapUpCmd = &cli.Command{
 	},
 }
 
-var sectorsSnapAbortCmd = &cli.Command{
-	Name:      "abort-upgrade",
-	Usage:     "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before",
-	ArgsUsage: "<sectorNum>",
-	Action: func(cctx *cli.Context) error {
-		if cctx.Args().Len() != 1 {
-			return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))
-		}
-
-		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer closer()
-		ctx := lcli.ReqContext(cctx)
-
-		id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
-		if err != nil {
-			return xerrors.Errorf("could not parse sector number: %w", err)
-		}
-
-		return nodeApi.SectorAbortUpgrade(ctx, abi.SectorNumber(id))
-	},
-}
-
 var sectorsMarkForUpgradeCmd = &cli.Command{
 	Name:      "mark-for-upgrade",
 	Usage:     "Mark a committed capacity sector for replacement by a sector with deals",

cmd/lotus-seal-worker/main.go

@@ -261,7 +261,7 @@ var runCmd = &cli.Command{
 
 		var taskTypes []sealtasks.TaskType
 
-		taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeReplicaUpdate)
+		taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize)
 
 		if cctx.Bool("addpiece") {
 			taskTypes = append(taskTypes, sealtasks.TTAddPiece)

cmd/lotus-shed/storage-stats.go

@@ -4,12 +4,10 @@ import (
 	"encoding/json"
 	corebig "math/big"
 	"os"
-	"strconv"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	filbig "github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
@@ -24,50 +22,29 @@ type networkTotalsOutput struct {
 	Payload  networkTotals `json:"payload"`
 }
 
-type providerMeta struct {
-	nonidentifiable bool
-}
-
-// force formatting as decimal to aid human readers
-type humanFloat float64
-
-func (f humanFloat) MarshalJSON() ([]byte, error) {
-	// 'f' uses decimal digits without exponents.
-	// The bit size of 32 ensures we don't use too many decimal places.
-	return []byte(strconv.FormatFloat(float64(f), 'f', -1, 32)), nil
-}
-
-type Totals struct {
+type networkTotals struct {
+	QaNetworkPower       filbig.Int `json:"total_qa_power"`
+	RawNetworkPower      filbig.Int `json:"total_raw_capacity"`
+	CapacityCarryingData float64    `json:"capacity_fraction_carrying_data"`
+	UniqueCids           int        `json:"total_unique_cids"`
+	UniqueProviders      int        `json:"total_unique_providers"`
+	UniqueClients        int        `json:"total_unique_clients"`
 	TotalDeals           int        `json:"total_num_deals"`
 	TotalBytes           int64      `json:"total_stored_data_size"`
-	PrivateTotalDeals    int        `json:"private_total_num_deals"`
-	PrivateTotalBytes    int64      `json:"private_total_stored_data_size"`
-	CapacityCarryingData humanFloat `json:"capacity_fraction_carrying_data"`
-}
+	FilplusTotalDeals    int        `json:"filplus_total_num_deals"`
+	FilplusTotalBytes    int64      `json:"filplus_total_stored_data_size"`
 
-type networkTotals struct {
-	QaNetworkPower         filbig.Int `json:"total_qa_power"`
-	RawNetworkPower        filbig.Int `json:"total_raw_capacity"`
-	UniqueCids             int        `json:"total_unique_cids"`
-	UniqueBytes            int64      `json:"total_unique_data_size"`
-	UniqueClients          int        `json:"total_unique_clients"`
-	UniqueProviders        int        `json:"total_unique_providers"`
-	UniquePrivateProviders int        `json:"total_unique_private_providers"`
-	Totals
-	FilPlus Totals `json:"filecoin_plus_subset"`
-
-	pieces    map[cid.Cid]struct{}
-	clients   map[address.Address]struct{}
-	providers map[address.Address]providerMeta
+	seenClient   map[address.Address]bool
+	seenProvider map[address.Address]bool
+	seenPieceCid map[cid.Cid]bool
 }
 
 var storageStatsCmd = &cli.Command{
 	Name:  "storage-stats",
 	Usage: "Translates current lotus state into a json summary suitable for driving https://storage.filecoin.io/",
 	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "tipset",
-			Usage: "Comma separated array of cids, or @height",
+		&cli.Int64Flag{
+			Name: "height",
 		},
 	},
 	Action: func(cctx *cli.Context) error {
@@ -79,24 +56,22 @@ var storageStatsCmd = &cli.Command{
 		}
 		defer apiCloser()
 
-		var ts *types.TipSet
-		if cctx.String("tipset") == "" {
-			ts, err = api.ChainHead(ctx)
-			if err != nil {
-				return err
-			}
-			ts, err = api.ChainGetTipSetByHeight(ctx, ts.Height()-defaultEpochLookback, ts.Key())
-			if err != nil {
-				return err
-			}
-		} else {
-			ts, err = lcli.ParseTipSetRef(ctx, api, cctx.String("tipset"))
-			if err != nil {
-				return err
-			}
+		head, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
 		}
 
-		power, err := api.StateMinerPower(ctx, address.Address{}, ts.Key())
+		requestedHeight := cctx.Int64("height")
+		if requestedHeight > 0 {
+			head, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(requestedHeight), head.Key())
+		} else {
+			head, err = api.ChainGetTipSetByHeight(ctx, head.Height()-defaultEpochLookback, head.Key())
+		}
+		if err != nil {
+			return err
+		}
+
+		power, err := api.StateMinerPower(ctx, address.Address{}, head.Key())
 		if err != nil {
 			return err
 		}
@@ -104,12 +79,12 @@ var storageStatsCmd = &cli.Command{
 		netTotals := networkTotals{
 			QaNetworkPower:  power.TotalPower.QualityAdjPower,
 			RawNetworkPower: power.TotalPower.RawBytePower,
-			pieces:          make(map[cid.Cid]struct{}),
-			clients:         make(map[address.Address]struct{}),
-			providers:       make(map[address.Address]providerMeta),
+			seenClient:      make(map[address.Address]bool),
+			seenProvider:    make(map[address.Address]bool),
+			seenPieceCid:    make(map[cid.Cid]bool),
 		}
 
-		deals, err := api.StateMarketDeals(ctx, ts.Key())
+		deals, err := api.StateMarketDeals(ctx, head.Key())
 		if err != nil {
 			return err
 		}
@@ -119,76 +94,35 @@ var storageStatsCmd = &cli.Command{
 			// Only count deals that have properly started, not past/future ones
 			// https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
 			// Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug
-			//
-			// Additionally if the SlashEpoch is set this means the underlying sector is
-			// terminated for whatever reason ( not just slashed ), and the deal record
-			// will soon be removed from the state entirely
 			if dealInfo.State.SectorStartEpoch <= 0 ||
-				dealInfo.State.SectorStartEpoch > ts.Height() ||
-				dealInfo.State.SlashEpoch > -1 {
+				dealInfo.State.SectorStartEpoch > head.Height() {
 				continue
 			}
 
-			netTotals.clients[dealInfo.Proposal.Client] = struct{}{}
-
-			if _, seen := netTotals.providers[dealInfo.Proposal.Provider]; !seen {
-				pm := providerMeta{}
-
-				mi, err := api.StateMinerInfo(ctx, dealInfo.Proposal.Provider, ts.Key())
-				if err != nil {
-					return err
-				}
-
-				if mi.PeerId == nil || *mi.PeerId == "" {
-					log.Infof("private provider %s", dealInfo.Proposal.Provider)
-					pm.nonidentifiable = true
-					netTotals.UniquePrivateProviders++
-				}
-
-				netTotals.providers[dealInfo.Proposal.Provider] = pm
-				netTotals.UniqueProviders++
-			}
-
-			if _, seen := netTotals.pieces[dealInfo.Proposal.PieceCID]; !seen {
-				netTotals.pieces[dealInfo.Proposal.PieceCID] = struct{}{}
-				netTotals.UniqueBytes += int64(dealInfo.Proposal.PieceSize)
-				netTotals.UniqueCids++
-			}
-
+			netTotals.seenClient[dealInfo.Proposal.Client] = true
 			netTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
+			netTotals.seenProvider[dealInfo.Proposal.Provider] = true
+			netTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
 			netTotals.TotalDeals++
-			if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable {
-				netTotals.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize)
-				netTotals.PrivateTotalDeals++
-			}
 
 			if dealInfo.Proposal.VerifiedDeal {
-				netTotals.FilPlus.TotalBytes += int64(dealInfo.Proposal.PieceSize)
-				netTotals.FilPlus.TotalDeals++
-				if netTotals.providers[dealInfo.Proposal.Provider].nonidentifiable {
-					netTotals.FilPlus.PrivateTotalBytes += int64(dealInfo.Proposal.PieceSize)
-					netTotals.FilPlus.PrivateTotalDeals++
-				}
+				netTotals.FilplusTotalDeals++
+				netTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
 			}
 		}
 
-		netTotals.UniqueClients = len(netTotals.clients)
+		netTotals.UniqueCids = len(netTotals.seenPieceCid)
+		netTotals.UniqueClients = len(netTotals.seenClient)
+		netTotals.UniqueProviders = len(netTotals.seenProvider)
 
-		ccd, _ := new(corebig.Rat).SetFrac(
+		netTotals.CapacityCarryingData, _ = new(corebig.Rat).SetFrac(
 			corebig.NewInt(netTotals.TotalBytes),
 			netTotals.RawNetworkPower.Int,
 		).Float64()
-		netTotals.CapacityCarryingData = humanFloat(ccd)
-
-		ccdfp, _ := new(corebig.Rat).SetFrac(
-			corebig.NewInt(netTotals.FilPlus.TotalBytes),
-			netTotals.RawNetworkPower.Int,
-		).Float64()
-		netTotals.FilPlus.CapacityCarryingData = humanFloat(ccdfp)
 
 		return json.NewEncoder(os.Stdout).Encode(
 			networkTotalsOutput{
-				Epoch:    int64(ts.Height()),
+				Epoch:    int64(head.Height()),
 				Endpoint: "NETWORK_WIDE_TOTALS",
 				Payload:  netTotals,
 			},

documentation/en/api-v0-methods-miner.md

@@ -23,6 +23,8 @@
   * [DagstoreInitializeAll](#DagstoreInitializeAll)
   * [DagstoreInitializeShard](#DagstoreInitializeShard)
   * [DagstoreListShards](#DagstoreListShards)
+  * [DagstoreLookupPieces](#DagstoreLookupPieces)
+  * [DagstorePieceIndexSize](#DagstorePieceIndexSize)
   * [DagstoreRecoverShard](#DagstoreRecoverShard)
 * [Deals](#Deals)
   * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
@@ -43,6 +45,9 @@
   * [DealsSetPieceCidBlocklist](#DealsSetPieceCidBlocklist)
 * [I](#I)
   * [ID](#ID)
+* [Indexer](#Indexer)
+  * [IndexerAnnounceAllDeals](#IndexerAnnounceAllDeals)
+  * [IndexerAnnounceDeal](#IndexerAnnounceDeal)
 * [Log](#Log)
   * [LogAlerts](#LogAlerts)
   * [LogList](#LogList)
@@ -84,6 +89,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -97,7 +105,6 @@
 * [Return](#Return)
   * [ReturnAddPiece](#ReturnAddPiece)
   * [ReturnFetch](#ReturnFetch)
-  * [ReturnFinalizeReplicaUpdate](#ReturnFinalizeReplicaUpdate)
   * [ReturnFinalizeSector](#ReturnFinalizeSector)
   * [ReturnGenerateSectorKeyFromData](#ReturnGenerateSectorKeyFromData)
   * [ReturnMoveStorage](#ReturnMoveStorage)
@@ -117,7 +124,6 @@
   * [SealingAbort](#SealingAbort)
   * [SealingSchedDiag](#SealingSchedDiag)
 * [Sector](#Sector)
-  * [SectorAbortUpgrade](#SectorAbortUpgrade)
   * [SectorAddPieceToAny](#SectorAddPieceToAny)
   * [SectorCommitFlush](#SectorCommitFlush)
   * [SectorCommitPending](#SectorCommitPending)
@@ -514,6 +520,42 @@ Response:
 ]
 ```
 
+### DagstoreLookupPieces
+DagstoreLookupPieces returns information about shards that contain the given CID.
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response:
+```json
+[
+  {
+    "Key": "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
+    "State": "ShardStateAvailable",
+    "Error": "\u003cerror\u003e"
+  }
+]
+```
+
+### DagstorePieceIndexSize
+DagstorePieceIndexSize returns the size of the piece index.
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `9`
+
 ### DagstoreRecoverShard
 DagstoreRecoverShard attempts to recover a failed shard.
 
@@ -772,6 +814,37 @@ Inputs: `null`
 
 Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"`
 
+## Indexer
+
+
+### IndexerAnnounceAllDeals
+IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `{}`
+
+### IndexerAnnounceDeal
+IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+so they can download its index
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `{}`
+
 ## Log
 
 
@@ -1784,6 +1857,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 
@@ -2056,30 +2175,6 @@ Response: `{}`
 ### ReturnFetch
 
 
-Perms: admin
-
-Inputs:
-```json
-[
-  {
-    "Sector": {
-      "Miner": 1000,
-      "Number": 9
-    },
-    "ID": "07070707-0707-0707-0707-070707070707"
-  },
-  {
-    "Code": 0,
-    "Message": "string value"
-  }
-]
-```
-
-Response: `{}`
-
-### ReturnFinalizeReplicaUpdate
-
-
 Perms: admin
 
 Inputs:
@@ -2500,21 +2595,6 @@ Response: `{}`
 ## Sector
 
 
-### SectorAbortUpgrade
-SectorAbortUpgrade can be called on sectors that are in the process of being upgraded to abort it
-
-
-Perms: admin
-
-Inputs:
-```json
-[
-  9
-]
-```
-
-Response: `{}`
-
 ### SectorAddPieceToAny
 Add piece to an open sector. If no sectors with enough space are open,
 either a new sector will be created, or this call will block until more

documentation/en/api-v0-methods-worker.md

@@ -10,7 +10,6 @@
 * [Add](#Add)
   * [AddPiece](#AddPiece)
 * [Finalize](#Finalize)
-  * [FinalizeReplicaUpdate](#FinalizeReplicaUpdate)
   * [FinalizeSector](#FinalizeSector)
 * [Generate](#Generate)
   * [GenerateSectorKeyFromData](#GenerateSectorKeyFromData)
@@ -1113,41 +1112,6 @@ Response:
 ## Finalize
 
 
-### FinalizeReplicaUpdate
-
-
-Perms: admin
-
-Inputs:
-```json
-[
-  {
-    "ID": {
-      "Miner": 1000,
-      "Number": 9
-    },
-    "ProofType": 8
-  },
-  [
-    {
-      "Offset": 1024,
-      "Size": 1024
-    }
-  ]
-]
-```
-
-Response:
-```json
-{
-  "Sector": {
-    "Miner": 1000,
-    "Number": 9
-  },
-  "ID": "07070707-0707-0707-0707-070707070707"
-}
-```
-
 ### FinalizeSector
 
 

documentation/en/api-v0-methods.md

@@ -131,6 +131,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -3905,6 +3908,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 

documentation/en/api-v1-unstable-methods.md

@@ -137,6 +137,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -4266,6 +4269,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-rc1
+   1.15.0-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -30,6 +30,7 @@ COMMANDS:
      retrieval-deals  Manage retrieval deals and related configuration
      data-transfers   Manage data transfers
      dagstore         Manage the dagstore on the markets subsystem
+     index            Manage the index provider on the markets subsystem
    NETWORK:
      net  Manage P2P Network
    RETRIEVAL:
@@ -78,7 +79,6 @@ OPTIONS:
    --gas-premium value          set gas premium for initialization messages in AttoFIL (default: "0")
    --from value                 select which address to send actor creation message from
    --help, -h                   show help (default: false)
-   --version, -v                print the version (default: false)
    
 ```
 
@@ -160,8 +160,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -203,11 +202,11 @@ USAGE:
 
 DESCRIPTION:
    The backup command writes a copy of node metadata under the specified path
-
-Online backups:
-For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
-to a path where backup files are supposed to be saved, and the path specified in
-this command must be within this base path
+   
+   Online backups:
+   For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
+   to a path where backup files are supposed to be saved, and the path specified in
+   this command must be within this base path
 
 OPTIONS:
    --offline   create backup without the node running (default: false)
@@ -249,8 +248,7 @@ COMMANDS:
    help, h                Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -339,8 +337,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -433,7 +430,6 @@ OPTIONS:
    --hide-sectors-info  hide sectors info (default: false)
    --blocks value       Log of produced <blocks> newest blocks and rewards(Miner Fee excluded) (default: 0)
    --help, -h           show help (default: false)
-   --version, -v        print the version (default: false)
    
 ```
 
@@ -464,8 +460,7 @@ COMMANDS:
    help, h       Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -512,8 +507,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -540,23 +534,22 @@ USAGE:
 
 DESCRIPTION:
    Set the log level for logging systems:
-
-   The system flag can be specified multiple times.
-
-   eg) log set-level --system chain --system chainxchg debug
-
-   Available Levels:
-   debug
-   info
-   warn
-   error
-
-   Environment Variables:
-   GOLOG_LOG_LEVEL - Default log level for all log systems
-   GOLOG_LOG_FMT   - Change output log format (json, nocolor)
-   GOLOG_FILE      - Write logs to file
-   GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
-
+   
+      The system flag can be specified multiple times.
+   
+      eg) log set-level --system chain --system chainxchg debug
+   
+      Available Levels:
+      debug
+      info
+      warn
+      error
+   
+      Environment Variables:
+      GOLOG_LOG_LEVEL - Default log level for all log systems
+      GOLOG_LOG_FMT   - Change output log format (json, nocolor)
+      GOLOG_FILE      - Write logs to file
+      GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
 
 OPTIONS:
    --system value  limit to log system
@@ -634,8 +627,7 @@ COMMANDS:
    help, h            Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -683,8 +675,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -856,8 +847,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -876,8 +866,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -981,8 +970,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1062,11 +1050,12 @@ COMMANDS:
    recover-shard     Attempt to recover a shard in errored state
    initialize-all    Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
    gc                Garbage collect the dagstore
+   piece-index-size  Inspect the dagstore piece index size
+   lookup-pieces     Lookup pieces that a given CID belongs to
    help, h           Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1142,6 +1131,80 @@ OPTIONS:
    
 ```
 
+### lotus-miner dagstore piece-index-size
+```
+NAME:
+   lotus-miner dagstore piece-index-size - Inspect the dagstore piece index size
+
+USAGE:
+   lotus-miner dagstore piece-index-size [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner dagstore lookup-pieces
+```
+NAME:
+   lotus-miner dagstore lookup-pieces - Lookup pieces that a given CID belongs to
+
+USAGE:
+   lotus-miner dagstore lookup-pieces [command options] <cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+## lotus-miner index
+```
+NAME:
+   lotus-miner index - Manage the index provider on the markets subsystem
+
+USAGE:
+   lotus-miner index command [command options] [arguments...]
+
+COMMANDS:
+   announce      Announce a deal to indexers so they can download its index
+   announce-all  Announce all active deals to indexers so they can download its indices
+   help, h       Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner index announce
+```
+NAME:
+   lotus-miner index announce - Announce a deal to indexers so they can download its index
+
+USAGE:
+   lotus-miner index announce [command options] <deal proposal cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner index announce-all
+```
+NAME:
+   lotus-miner index announce-all - Announce all active deals to indexers so they can download its indices
+
+USAGE:
+   lotus-miner index announce-all [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner net
 ```
 NAME:
@@ -1151,22 +1214,24 @@ USAGE:
    lotus-miner net command [command options] [arguments...]
 
 COMMANDS:
-   peers         Print peers
-   connect       Connect to a peer
-   listen        List listen addresses
-   id            Get node identity
-   findpeer      Find the addresses of a given peerID
-   scores        Print peers' pubsub scores
-   reachability  Print information about reachability from the internet
-   bandwidth     Print bandwidth usage information
-   block         Manage network connection gating rules
-   stat          Report resource usage for a scope
-   limit         Get or set resource limits for a scope
-   help, h       Shows a list of commands or help for one command
+   peers           Print peers
+   connect         Connect to a peer
+   listen          List listen addresses
+   id              Get node identity
+   findpeer        Find the addresses of a given peerID
+   scores          Print peers' pubsub scores
+   reachability    Print information about reachability from the internet
+   bandwidth       Print bandwidth usage information
+   block           Manage network connection gating rules
+   stat            Report resource usage for a scope
+   limit           Get or set resource limits for a scope
+   protect         Add one or more peer IDs to the list of protected peer connections
+   unprotect       Remove one or more peer IDs from the list of protected peer connections.
+   list-protected  List the peer IDs with protected connection.
+   help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1294,8 +1359,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1314,8 +1378,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1373,8 +1436,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1440,15 +1502,14 @@ USAGE:
 
 DESCRIPTION:
    Report resource usage for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-  - all           -- reports the resource usage for all currently active scopes.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+     - all           -- reports the resource usage for all currently active scopes.
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -1465,16 +1526,15 @@ USAGE:
 
 DESCRIPTION:
    Get or set resource limits for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-
- The limit is json-formatted, with the same structure as the limits file.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+   
+    The limit is json-formatted, with the same structure as the limits file.
 
 OPTIONS:
    --set       set the limit for a scope (default: false)
@@ -1482,6 +1542,45 @@ OPTIONS:
    
 ```
 
+### lotus-miner net protect
+```
+NAME:
+   lotus-miner net protect - Add one or more peer IDs to the list of protected peer connections
+
+USAGE:
+   lotus-miner net protect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner net unprotect
+```
+NAME:
+   lotus-miner net unprotect - Remove one or more peer IDs from the list of protected peer connections.
+
+USAGE:
+   lotus-miner net unprotect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner net list-protected
+```
+NAME:
+   lotus-miner net list-protected - List the peer IDs with protected connection.
+
+USAGE:
+   lotus-miner net list-protected [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner pieces
 ```
 NAME:
@@ -1501,8 +1600,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1580,7 +1678,6 @@ COMMANDS:
    terminate             Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector)
    remove                Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty))
    snap-up               Mark a committed capacity sector to be filled with deals
-   abort-upgrade         Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before
    mark-for-upgrade      Mark a committed capacity sector for replacement by a sector with deals
    seal                  Manually start sealing a sector (filling any unused space with junk)
    set-seal-delay        Set the time, in minutes, that a new sector waits for deals before sealing starts
@@ -1590,8 +1687,7 @@ COMMANDS:
    help, h               Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1759,7 +1855,6 @@ COMMANDS:
 OPTIONS:
    --really-do-it  pass this flag if you know what you are doing (default: false)
    --help, -h      show help (default: false)
-   --version, -v   print the version (default: false)
    
 ```
 
@@ -1816,19 +1911,6 @@ OPTIONS:
    
 ```
 
-### lotus-miner sectors abort-upgrade
-```
-NAME:
-   lotus-miner sectors abort-upgrade - Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before
-
-USAGE:
-   lotus-miner sectors abort-upgrade [command options] <sectorNum>
-
-OPTIONS:
-   --help, -h  show help (default: false)
-   
-```
-
 ### lotus-miner sectors mark-for-upgrade
 ```
 NAME:
@@ -1896,8 +1978,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1959,8 +2040,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2042,9 +2122,9 @@ USAGE:
 
 DESCRIPTION:
    Sectors can be stored across many filesystem paths. These
-commands provide ways to manage the storage the miner will used to store sectors
-long term for proving (references as 'store') as well as how sectors will be
-stored while moving through the sealing pipeline (references as 'seal').
+   commands provide ways to manage the storage the miner will used to store sectors
+   long term for proving (references as 'store') as well as how sectors will be
+   stored while moving through the sealing pipeline (references as 'seal').
 
 COMMANDS:
    attach   attach local storage path
@@ -2055,8 +2135,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2070,24 +2149,23 @@ USAGE:
 
 DESCRIPTION:
    Storage can be attached to the miner using this command. The storage volume
-list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not
-recommend manually modifying this value without further understanding of the
-storage system.
-
-Each storage volume contains a configuration file which describes the
-capabilities of the volume. When the '--init' flag is provided, this file will
-be created using the additional flags.
-
-Weight
-A high weight value means data will be more likely to be stored in this path
-
-Seal
-Data for the sealing process will be stored here
-
-Store
-Finalized sectors that will be moved here for long term storage and be proven
-over time
+   list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not
+   recommend manually modifying this value without further understanding of the
+   storage system.
    
+   Each storage volume contains a configuration file which describes the
+   capabilities of the volume. When the '--init' flag is provided, this file will
+   be created using the additional flags.
+   
+   Weight
+   A high weight value means data will be more likely to be stored in this path
+   
+   Seal
+   Data for the sealing process will be stored here
+   
+   Store
+   Finalized sectors that will be moved here for long term storage and be proven
+   over time
 
 OPTIONS:
    --init               initialize the path first (default: false)
@@ -2114,9 +2192,8 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --color        use color in display output (default: depends on output being a TTY)
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2190,8 +2267,7 @@ COMMANDS:
    help, h     Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-rc1
+   1.15.0-dev
 
 COMMANDS:
    run         Start lotus worker
@@ -78,8 +78,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -159,8 +158,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-rc1
+   1.15.0-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process
@@ -71,7 +71,6 @@ OPTIONS:
    --restore value           restore from backup file
    --restore-config value    config file to use when restoring from backup
    --help, -h                show help (default: false)
-   --version, -v             print the version (default: false)
    
 ```
 
@@ -98,11 +97,11 @@ USAGE:
 
 DESCRIPTION:
    The backup command writes a copy of node metadata under the specified path
-
-Online backups:
-For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
-to a path where backup files are supposed to be saved, and the path specified in
-this command must be within this base path
+   
+   Online backups:
+   For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
+   to a path where backup files are supposed to be saved, and the path specified in
+   this command must be within this base path
 
 OPTIONS:
    --offline   create backup without the node running (default: false)
@@ -124,8 +123,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -218,8 +216,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -372,8 +369,7 @@ COMMANDS:
    help, h   Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -448,8 +444,7 @@ COMMANDS:
      cancel-transfer   Force cancel a data transfer
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -566,37 +561,36 @@ CATEGORY:
 
 DESCRIPTION:
    Retrieve data from the Filecoin network.
-
-The retrieve command will attempt to find a provider make a retrieval deal with
-them. In case a provider can't be found, it can be specified with the --provider
-flag.
-
-By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
-a CAR file containing the raw IPLD graph can be exported by setting the --car
-flag.
-
-Partial Retrieval:
-
-The --data-selector flag can be used to specify a sub-graph to fetch. The
-selector can be specified as either IPLD datamodel text-path selector, or IPLD
-json selector.
-
-In case of unixfs retrieval, the selector must point at a single root node, and
-match the entire graph under that node.
-
-In case of CAR retrieval, the selector must have one common "sub-root" node.
-
-Examples:
-
-- Retrieve a file by CID
-  $ lotus client retrieve Qm... my-file.txt
-
-- Retrieve a file by CID from f0123
-  $ lotus client retrieve --provider f0123 Qm... my-file.txt
-
-- Retrieve a first file from a specified directory
-  $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
-
+   
+   The retrieve command will attempt to find a provider make a retrieval deal with
+   them. In case a provider can't be found, it can be specified with the --provider
+   flag.
+   
+   By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
+   a CAR file containing the raw IPLD graph can be exported by setting the --car
+   flag.
+   
+   Partial Retrieval:
+   
+   The --data-selector flag can be used to specify a sub-graph to fetch. The
+   selector can be specified as either IPLD datamodel text-path selector, or IPLD
+   json selector.
+   
+   In case of unixfs retrieval, the selector must point at a single root node, and
+   match the entire graph under that node.
+   
+   In case of CAR retrieval, the selector must have one common "sub-root" node.
+   
+   Examples:
+   
+   - Retrieve a file by CID
+     $ lotus client retrieve Qm... my-file.txt
+   
+   - Retrieve a file by CID from f0123
+     $ lotus client retrieve --provider f0123 Qm... my-file.txt
+   
+   - Retrieve a first file from a specified directory
+     $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
 
 OPTIONS:
    --car                                                   Export to a car file instead of a regular file (default: false)
@@ -709,13 +703,13 @@ CATEGORY:
 
 DESCRIPTION:
    Make a deal with a miner.
-dataCid comes from running 'lotus client import'.
-miner is the address of the miner you wish to make a deal with.
-price is measured in FIL/Epoch. Miners usually don't accept a bid
-lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
-with 'lotus client query-ask <miner address>'.
-duration is how long the miner should store the data for, in blocks.
-The minimum value is 518400 (6 months).
+   dataCid comes from running 'lotus client import'.
+   miner is the address of the miner you wish to make a deal with.
+   price is measured in FIL/Epoch. Miners usually don't accept a bid
+   lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
+   with 'lotus client query-ask <miner address>'.
+   duration is how long the miner should store the data for, in blocks.
+   The minimum value is 518400 (6 months).
 
 OPTIONS:
    --manual-piece-cid value     manually specify piece commitment for data (dataCid must be to a car file)
@@ -976,7 +970,6 @@ COMMANDS:
 OPTIONS:
    --confidence value  number of block confirmations to wait for (default: 5)
    --help, -h          show help (default: false)
-   --version, -v       print the version (default: false)
    
 ```
 
@@ -1242,8 +1235,7 @@ COMMANDS:
    help, h               Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1332,8 +1324,7 @@ COMMANDS:
    help, h            Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1382,8 +1373,7 @@ COMMANDS:
    help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1534,8 +1524,7 @@ COMMANDS:
    help, h       Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1587,8 +1576,7 @@ COMMANDS:
    help, h   Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1748,7 +1736,6 @@ COMMANDS:
 OPTIONS:
    --tipset value  specify tipset to call method on (pass comma separated array of cids)
    --help, -h      show help (default: false)
-   --version, -v   print the version (default: false)
    
 ```
 
@@ -2033,8 +2020,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2119,8 +2105,7 @@ COMMANDS:
    help, h          Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2191,10 +2176,9 @@ USAGE:
 
 DESCRIPTION:
    Collect object size and ipld link count for an object.
-
-   When a base is provided it will be walked first, and all links visisted
-   will be ignored when the passed in object is walked.
-
+   
+      When a base is provided it will be walked first, and all links visisted
+      will be ignored when the passed in object is walked.
 
 OPTIONS:
    --base value  ignore links found in this obj
@@ -2244,34 +2228,33 @@ USAGE:
 
 DESCRIPTION:
    Get ipld node under a specified path:
-
-   lotus chain get /ipfs/[cid]/some/path
-
-   Path prefixes:
-   - /ipfs/[cid], /ipld/[cid] - traverse IPLD path
-   - /pstate - traverse from head.ParentStateRoot
-
-   Note:
-   You can use special path elements to traverse through some data structures:
-   - /ipfs/[cid]/@H:elem - get 'elem' from hamt
-   - /ipfs/[cid]/@Hi:123 - get varint elem 123 from hamt
-   - /ipfs/[cid]/@Hu:123 - get uvarint elem 123 from hamt
-   - /ipfs/[cid]/@Ha:t01 - get element under Addr(t01).Bytes
-   - /ipfs/[cid]/@A:10   - get 10th amt element
-   - .../@Ha:t01/@state  - get pretty map-based actor state
-
-   List of --as-type types:
-   - raw
-   - block
-   - message
-   - smessage, signedmessage
-   - actor
-   - amt
-   - hamt-epoch
-   - hamt-address
-   - cronevent
-   - account-state
-
+   
+      lotus chain get /ipfs/[cid]/some/path
+   
+      Path prefixes:
+      - /ipfs/[cid], /ipld/[cid] - traverse IPLD path
+      - /pstate - traverse from head.ParentStateRoot
+   
+      Note:
+      You can use special path elements to traverse through some data structures:
+      - /ipfs/[cid]/@H:elem - get 'elem' from hamt
+      - /ipfs/[cid]/@Hi:123 - get varint elem 123 from hamt
+      - /ipfs/[cid]/@Hu:123 - get uvarint elem 123 from hamt
+      - /ipfs/[cid]/@Ha:t01 - get element under Addr(t01).Bytes
+      - /ipfs/[cid]/@A:10   - get 10th amt element
+      - .../@Ha:t01/@state  - get pretty map-based actor state
+   
+      List of --as-type types:
+      - raw
+      - block
+      - message
+      - smessage, signedmessage
+      - actor
+      - amt
+      - hamt-epoch
+      - hamt-address
+      - cronevent
+      - account-state
 
 OPTIONS:
    --as-type value  specify type to interpret output as
@@ -2291,18 +2274,17 @@ USAGE:
 
 DESCRIPTION:
    Bisect the chain state tree:
-
-   lotus chain bisect [min height] [max height] '1/2/3/state/path' 'shell command' 'args'
-
-   Returns the first tipset in which condition is true
-                  v
-   [start] FFFFFFFTTT [end]
-
-   Example: find height at which deal ID 100 000 appeared
-    - lotus chain bisect 1 32000 '@Ha:t03/1' jq -e '.[2] > 100000'
-
-   For special path elements see 'chain get' help
-
+   
+      lotus chain bisect [min height] [max height] '1/2/3/state/path' 'shell command' 'args'
+   
+      Returns the first tipset in which condition is true
+                     v
+      [start] FFFFFFFTTT [end]
+   
+      Example: find height at which deal ID 100 000 appeared
+       - lotus chain bisect 1 32000 '@Ha:t03/1' jq -e '.[2] > 100000'
+   
+      For special path elements see 'chain get' help
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -2382,8 +2364,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2415,8 +2396,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2453,7 +2433,6 @@ OPTIONS:
    --max-fee value  Spend up to X FIL per DisputeWindowedPoSt message
    --from value     optionally specify the account to send messages from
    --help, -h       show help (default: false)
-   --version, -v    print the version (default: false)
    
 ```
 
@@ -2499,8 +2478,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2527,23 +2505,22 @@ USAGE:
 
 DESCRIPTION:
    Set the log level for logging systems:
-
-   The system flag can be specified multiple times.
-
-   eg) log set-level --system chain --system chainxchg debug
-
-   Available Levels:
-   debug
-   info
-   warn
-   error
-
-   Environment Variables:
-   GOLOG_LOG_LEVEL - Default log level for all log systems
-   GOLOG_LOG_FMT   - Change output log format (json, nocolor)
-   GOLOG_FILE      - Write logs to file
-   GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
-
+   
+      The system flag can be specified multiple times.
+   
+      eg) log set-level --system chain --system chainxchg debug
+   
+      Available Levels:
+      debug
+      info
+      warn
+      error
+   
+      Environment Variables:
+      GOLOG_LOG_LEVEL - Default log level for all log systems
+      GOLOG_LOG_FMT   - Change output log format (json, nocolor)
+      GOLOG_FILE      - Write logs to file
+      GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
 
 OPTIONS:
    --system value  limit to log system
@@ -2607,22 +2584,24 @@ USAGE:
    lotus net command [command options] [arguments...]
 
 COMMANDS:
-   peers         Print peers
-   connect       Connect to a peer
-   listen        List listen addresses
-   id            Get node identity
-   findpeer      Find the addresses of a given peerID
-   scores        Print peers' pubsub scores
-   reachability  Print information about reachability from the internet
-   bandwidth     Print bandwidth usage information
-   block         Manage network connection gating rules
-   stat          Report resource usage for a scope
-   limit         Get or set resource limits for a scope
-   help, h       Shows a list of commands or help for one command
+   peers           Print peers
+   connect         Connect to a peer
+   listen          List listen addresses
+   id              Get node identity
+   findpeer        Find the addresses of a given peerID
+   scores          Print peers' pubsub scores
+   reachability    Print information about reachability from the internet
+   bandwidth       Print bandwidth usage information
+   block           Manage network connection gating rules
+   stat            Report resource usage for a scope
+   limit           Get or set resource limits for a scope
+   protect         Add one or more peer IDs to the list of protected peer connections
+   unprotect       Remove one or more peer IDs from the list of protected peer connections.
+   list-protected  List the peer IDs with protected connection.
+   help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2750,8 +2729,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2770,8 +2748,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2829,8 +2806,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2896,15 +2872,14 @@ USAGE:
 
 DESCRIPTION:
    Report resource usage for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-  - all           -- reports the resource usage for all currently active scopes.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+     - all           -- reports the resource usage for all currently active scopes.
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -2921,16 +2896,15 @@ USAGE:
 
 DESCRIPTION:
    Get or set resource limits for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-
- The limit is json-formatted, with the same structure as the limits file.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+   
+    The limit is json-formatted, with the same structure as the limits file.
 
 OPTIONS:
    --set       set the limit for a scope (default: false)
@@ -2938,6 +2912,45 @@ OPTIONS:
    
 ```
 
+### lotus net protect
+```
+NAME:
+   lotus net protect - Add one or more peer IDs to the list of protected peer connections
+
+USAGE:
+   lotus net protect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus net unprotect
+```
+NAME:
+   lotus net unprotect - Remove one or more peer IDs from the list of protected peer connections.
+
+USAGE:
+   lotus net unprotect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus net list-protected
+```
+NAME:
+   lotus net list-protected - List the peer IDs with protected connection.
+
+USAGE:
+   lotus net list-protected [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus sync
 ```
 NAME:
@@ -2956,8 +2969,7 @@ COMMANDS:
    help, h     Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 

documentation/en/default-lotus-config.toml

@@ -163,11 +163,11 @@
     #HotStoreType = "badger"
 
     # MarkSetType specifies the type of the markset.
-    # It can be "map" for in memory marking or "badger" (default) for on-disk marking.
+    # It can be "map" (default) for in memory marking or "badger" for on-disk marking.
     #
     # type: string
     # env var: LOTUS_CHAINSTORE_SPLITSTORE_MARKSETTYPE
-    #MarkSetType = "badger"
+    #MarkSetType = "map"
 
     # HotStoreMessageRetention specifies the retention policy for messages, in finalities beyond
     # the compaction boundary; default is 0.

documentation/en/default-lotus-miner-config.toml

@@ -167,14 +167,6 @@
   # env var: LOTUS_DEALMAKING_EXPECTEDSEALDURATION
   #ExpectedSealDuration = "24h0m0s"
 
-  # Whether new sectors are created to pack incoming deals
-  # When this is set to false no new sectors will be created for sealing incoming deals
-  # This is useful for forcing all deals to be assigned as snap deals to sectors marked for upgrade
-  #
-  # type: bool
-  # env var: LOTUS_DEALMAKING_MAKENEWSECTORFORDEALS
-  #MakeNewSectorForDeals = true
-
   # Maximum amount of time proposed deal StartEpoch can be in future
   #
   # type: Duration
@@ -265,6 +257,51 @@
       #Path = ""
 
 
+[IndexProvider]
+  # env var: LOTUS_INDEXPROVIDER_LINKCACHESIZE
+  #LinkCacheSize = 1024
+
+  # env var: LOTUS_INDEXPROVIDER_LINKEDCHUNKSIZE
+  #LinkedChunkSize = 16384
+
+  # env var: LOTUS_INDEXPROVIDER_PUBSUBTOPIC
+  #PubSubTopic = "/indexer/ingest/mainnet"
+
+  # env var: LOTUS_INDEXPROVIDER_PURGELINKCACHE
+  #PurgeLinkCache = false
+
+  # env var: LOTUS_INDEXPROVIDER_PUBLISHERKIND
+  #PublisherKind = "dtsync"
+
+  # Binding address for the libp2p host contacted by indexer nodes to sync the list of advertised
+  # multihashes. Note that when port is set to 0 a random port is generated at runtime and may be
+  # different on every restart. The format of the strings specified must conform to multiaddress;
+  # see https://multiformats.io/multiaddr/
+  #
+  # type: []string
+  # env var: LOTUS_INDEXPROVIDER_LISTENADDRESSES
+  #ListenAddresses = ["/ip4/0.0.0.0/tcp/0", "/ip6/::/tcp/0"]
+
+  # The address the endpoints at which the data associated to the advertised
+  # multihashes can be retrieved. If not specified, the ListenAddresses are used instead. The format
+  # of the strings specified must conform to multiaddress; see https://multiformats.io/multiaddr/
+  #
+  # type: []string
+  # env var: LOTUS_INDEXPROVIDER_ANNOUNCEADDRESSES
+  #AnnounceAddresses = []
+
+  # The maximum number of simultaneous requests syncing the list of advertised multihashes between
+  # the indexers and the index provider.
+  #
+  # type: uint64
+  # env var: LOTUS_INDEXPROVIDER_MAXSIMULTANEOUSTRANSFERS
+  #MaxSimultaneousTransfers = 20
+
+  [IndexProvider.HttpPublisher]
+    # env var: LOTUS_INDEXPROVIDER_HTTPPUBLISHER_LISTENMULTIADDR
+    #ListenMultiaddr = "/ip4/0.0.0.0/tcp/3104/http"
+
+
 [Sealing]
   # Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time.
   # If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created.

extern/sector-storage/faults.go

@@ -55,25 +55,6 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 				return nil
 			}
 
-			// temporary hack to make the check work with snapdeals
-			// will go away in https://github.com/filecoin-project/lotus/pull/7971
-			if lp.Sealed == "" || lp.Cache == "" {
-				// maybe it's update
-				lockedUpdate, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone)
-				if err != nil {
-					return xerrors.Errorf("acquiring sector lock: %w", err)
-				}
-				if lockedUpdate {
-					lp, _, err = m.localStore.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
-					if err != nil {
-						log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-						bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
-						return nil
-					}
-					lp.Sealed, lp.Cache = lp.Update, lp.UpdateCache
-				}
-			}
-
 			if lp.Sealed == "" || lp.Cache == "" {
 				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
 				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed)

extern/sector-storage/ffiwrapper/sealer_cgo.go

@@ -769,7 +769,7 @@ func (sb *Sealer) ReleaseSealed(ctx context.Context, sector storage.SectorRef) e
 	return xerrors.Errorf("not supported at this layer")
 }
 
-func (sb *Sealer) freeUnsealed(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
+func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
 	ssize, err := sector.ProofType.SectorSize()
 	if err != nil {
 		return err
@@ -834,19 +834,6 @@ func (sb *Sealer) freeUnsealed(ctx context.Context, sector storage.SectorRef, ke
 
 	}
 
-	return nil
-}
-
-func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
-	ssize, err := sector.ProofType.SectorSize()
-	if err != nil {
-		return err
-	}
-
-	if err := sb.freeUnsealed(ctx, sector, keepUnsealed); err != nil {
-		return err
-	}
-
 	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache, 0, storiface.PathStorage)
 	if err != nil {
 		return xerrors.Errorf("acquiring sector cache path: %w", err)
@@ -856,43 +843,6 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef,
 	return ffi.ClearCache(uint64(ssize), paths.Cache)
 }
 
-func (sb *Sealer) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
-	ssize, err := sector.ProofType.SectorSize()
-	if err != nil {
-		return err
-	}
-
-	if err := sb.freeUnsealed(ctx, sector, keepUnsealed); err != nil {
-		return err
-	}
-
-	{
-		paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache, 0, storiface.PathStorage)
-		if err != nil {
-			return xerrors.Errorf("acquiring sector cache path: %w", err)
-		}
-		defer done()
-
-		if err := ffi.ClearCache(uint64(ssize), paths.Cache); err != nil {
-			return xerrors.Errorf("clear cache: %w", err)
-		}
-	}
-
-	{
-		paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdateCache, 0, storiface.PathStorage)
-		if err != nil {
-			return xerrors.Errorf("acquiring sector cache path: %w", err)
-		}
-		defer done()
-
-		if err := ffi.ClearCache(uint64(ssize), paths.UpdateCache); err != nil {
-			return xerrors.Errorf("clear cache: %w", err)
-		}
-	}
-
-	return nil
-}
-
 func (sb *Sealer) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) error {
 	// This call is meant to mark storage as 'freeable'. Given that unsealing is
 	// very expensive, we don't remove data as soon as we can - instead we only

extern/sector-storage/manager.go

@@ -146,7 +146,7 @@ func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls store
 	go m.sched.runSched()
 
 	localTasks := []sealtasks.TaskType{
-		sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFetch, sealtasks.TTFinalizeReplicaUpdate,
+		sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFetch,
 	}
 	if sc.AllowAddPiece {
 		localTasks = append(localTasks, sealtasks.TTAddPiece)
@@ -577,74 +577,6 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storage.SectorRef,
 	return nil
 }
 
-func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	if err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache); err != nil {
-		return xerrors.Errorf("acquiring sector lock: %w", err)
-	}
-
-	fts := storiface.FTUnsealed
-	{
-		unsealedStores, err := m.index.StorageFindSector(ctx, sector.ID, storiface.FTUnsealed, 0, false)
-		if err != nil {
-			return xerrors.Errorf("finding unsealed sector: %w", err)
-		}
-
-		if len(unsealedStores) == 0 { // Is some edge-cases unsealed sector may not exist already, that's fine
-			fts = storiface.FTNone
-		}
-	}
-
-	pathType := storiface.PathStorage
-	{
-		sealedStores, err := m.index.StorageFindSector(ctx, sector.ID, storiface.FTUpdate, 0, false)
-		if err != nil {
-			return xerrors.Errorf("finding sealed sector: %w", err)
-		}
-
-		for _, store := range sealedStores {
-			if store.CanSeal {
-				pathType = storiface.PathSealing
-				break
-			}
-		}
-	}
-
-	selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed|storiface.FTUpdate|storiface.FTUpdateCache, false)
-
-	err := m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeReplicaUpdate, selector,
-		m.schedFetch(sector, storiface.FTCache|storiface.FTSealed|storiface.FTUpdate|storiface.FTUpdateCache|fts, pathType, storiface.AcquireMove),
-		func(ctx context.Context, w Worker) error {
-			_, err := m.waitSimpleCall(ctx)(w.FinalizeReplicaUpdate(ctx, sector, keepUnsealed))
-			return err
-		})
-	if err != nil {
-		return err
-	}
-
-	fetchSel := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathStorage)
-	moveUnsealed := fts
-	{
-		if len(keepUnsealed) == 0 {
-			moveUnsealed = storiface.FTNone
-		}
-	}
-
-	err = m.sched.Schedule(ctx, sector, sealtasks.TTFetch, fetchSel,
-		m.schedFetch(sector, storiface.FTCache|storiface.FTSealed|storiface.FTUpdate|storiface.FTUpdateCache|moveUnsealed, storiface.PathStorage, storiface.AcquireMove),
-		func(ctx context.Context, w Worker) error {
-			_, err := m.waitSimpleCall(ctx)(w.MoveStorage(ctx, sector, storiface.FTCache|storiface.FTSealed|storiface.FTUpdate|storiface.FTUpdateCache|moveUnsealed))
-			return err
-		})
-	if err != nil {
-		return xerrors.Errorf("moving sector to storage: %w", err)
-	}
-
-	return nil
-}
-
 func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) error {
 	return nil
 }
@@ -943,10 +875,6 @@ func (m *Manager) ReturnProveReplicaUpdate2(ctx context.Context, callID storifac
 	return m.returnResult(ctx, callID, proof, err)
 }
 
-func (m *Manager) ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error {
-	return m.returnResult(ctx, callID, nil, err)
-}
-
 func (m *Manager) ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error {
 	return m.returnResult(ctx, callID, nil, err)
 }

extern/sector-storage/mock/mock.go

@@ -477,10 +477,6 @@ func (mgr *SectorMgr) FinalizeSector(context.Context, storage.SectorRef, []stora
 	return nil
 }
 
-func (mgr *SectorMgr) FinalizeReplicaUpdate(context.Context, storage.SectorRef, []storage.Range) error {
-	return nil
-}
-
 func (mgr *SectorMgr) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) error {
 	return nil
 }
@@ -581,10 +577,6 @@ func (mgr *SectorMgr) ReturnGenerateSectorKeyFromData(ctx context.Context, callI
 	panic("not supported")
 }
 
-func (mgr *SectorMgr) ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error {
-	panic("not supported")
-}
-
 func (m mockVerifProver) VerifySeal(svi proof.SealVerifyInfo) (bool, error) {
 	plen, err := svi.SealProof.ProofSize()
 	if err != nil {

extern/sector-storage/sched_test.go

@@ -119,10 +119,6 @@ func (s *schedTestWorker) GenerateSectorKeyFromData(ctx context.Context, sector
 	panic("implement me")
 }
 
-func (s *schedTestWorker) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) {
-	panic("implement me")
-}
-
 func (s *schedTestWorker) MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) {
 	panic("implement me")
 }

extern/sector-storage/sealtasks/task.go

@@ -14,11 +14,10 @@ const (
 	TTFetch  TaskType = "seal/v0/fetch"
 	TTUnseal TaskType = "seal/v0/unseal"
 
-	TTReplicaUpdate         TaskType = "seal/v0/replicaupdate"
-	TTProveReplicaUpdate1   TaskType = "seal/v0/provereplicaupdate/1"
-	TTProveReplicaUpdate2   TaskType = "seal/v0/provereplicaupdate/2"
-	TTRegenSectorKey        TaskType = "seal/v0/regensectorkey"
-	TTFinalizeReplicaUpdate TaskType = "seal/v0/finalize/replicaupdate"
+	TTReplicaUpdate       TaskType = "seal/v0/replicaupdate"
+	TTProveReplicaUpdate1 TaskType = "seal/v0/provereplicaupdate/1"
+	TTProveReplicaUpdate2 TaskType = "seal/v0/provereplicaupdate/2"
+	TTRegenSectorKey      TaskType = "seal/v0/regensectorkey"
 )
 
 var order = map[TaskType]int{
@@ -49,11 +48,10 @@ var shortNames = map[TaskType]string{
 	TTFetch:  "GET",
 	TTUnseal: "UNS",
 
-	TTReplicaUpdate:         "RU",
-	TTProveReplicaUpdate1:   "PR1",
-	TTProveReplicaUpdate2:   "PR2",
-	TTRegenSectorKey:        "GSK",
-	TTFinalizeReplicaUpdate: "FRU",
+	TTReplicaUpdate:       "RU",
+	TTProveReplicaUpdate1: "PR1",
+	TTProveReplicaUpdate2: "PR2",
+	TTRegenSectorKey:      "GSK",
 }
 
 func (a TaskType) MuchLess(b TaskType) (bool, bool) {

extern/sector-storage/storiface/worker.go

@@ -120,7 +120,6 @@ type WorkerCalls interface {
 	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (CallID, error)
 	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (CallID, error)
 	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (CallID, error)
-	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (CallID, error)
 	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (CallID, error)
 	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (CallID, error)
 	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (CallID, error)
@@ -183,7 +182,6 @@ type WorkerReturn interface {
 	ReturnProveReplicaUpdate1(ctx context.Context, callID CallID, proofs storage.ReplicaVanillaProofs, err *CallError) error
 	ReturnProveReplicaUpdate2(ctx context.Context, callID CallID, proof storage.ReplicaUpdateProof, err *CallError) error
 	ReturnGenerateSectorKeyFromData(ctx context.Context, callID CallID, err *CallError) error
-	ReturnFinalizeReplicaUpdate(ctx context.Context, callID CallID, err *CallError) error
 	ReturnMoveStorage(ctx context.Context, callID CallID, err *CallError) error
 	ReturnUnsealPiece(ctx context.Context, callID CallID, err *CallError) error
 	ReturnReadPiece(ctx context.Context, callID CallID, ok bool, err *CallError) error

extern/sector-storage/teststorage_test.go

@@ -87,10 +87,6 @@ func (t *testExec) GenerateSectorKeyFromData(ctx context.Context, sector storage
 	panic("implement me")
 }
 
-func (t *testExec) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error {
-	panic("implement me")
-}
-
 func (t *testExec) NewSector(ctx context.Context, sector storage.SectorRef) error {
 	panic("implement me")
 }

extern/sector-storage/worker_local.go

@@ -162,21 +162,20 @@ func (l *LocalWorker) ffiExec() (ffiwrapper.Storage, error) {
 type ReturnType string
 
 const (
-	AddPiece              ReturnType = "AddPiece"
-	SealPreCommit1        ReturnType = "SealPreCommit1"
-	SealPreCommit2        ReturnType = "SealPreCommit2"
-	SealCommit1           ReturnType = "SealCommit1"
-	SealCommit2           ReturnType = "SealCommit2"
-	FinalizeSector        ReturnType = "FinalizeSector"
-	FinalizeReplicaUpdate ReturnType = "FinalizeReplicaUpdate"
-	ReplicaUpdate         ReturnType = "ReplicaUpdate"
-	ProveReplicaUpdate1   ReturnType = "ProveReplicaUpdate1"
-	ProveReplicaUpdate2   ReturnType = "ProveReplicaUpdate2"
-	GenerateSectorKey     ReturnType = "GenerateSectorKey"
-	ReleaseUnsealed       ReturnType = "ReleaseUnsealed"
-	MoveStorage           ReturnType = "MoveStorage"
-	UnsealPiece           ReturnType = "UnsealPiece"
-	Fetch                 ReturnType = "Fetch"
+	AddPiece            ReturnType = "AddPiece"
+	SealPreCommit1      ReturnType = "SealPreCommit1"
+	SealPreCommit2      ReturnType = "SealPreCommit2"
+	SealCommit1         ReturnType = "SealCommit1"
+	SealCommit2         ReturnType = "SealCommit2"
+	FinalizeSector      ReturnType = "FinalizeSector"
+	ReplicaUpdate       ReturnType = "ReplicaUpdate"
+	ProveReplicaUpdate1 ReturnType = "ProveReplicaUpdate1"
+	ProveReplicaUpdate2 ReturnType = "ProveReplicaUpdate2"
+	GenerateSectorKey   ReturnType = "GenerateSectorKey"
+	ReleaseUnsealed     ReturnType = "ReleaseUnsealed"
+	MoveStorage         ReturnType = "MoveStorage"
+	UnsealPiece         ReturnType = "UnsealPiece"
+	Fetch               ReturnType = "Fetch"
 )
 
 // in: func(WorkerReturn, context.Context, CallID, err string)
@@ -214,21 +213,20 @@ func rfunc(in interface{}) func(context.Context, storiface.CallID, storiface.Wor
 }
 
 var returnFunc = map[ReturnType]func(context.Context, storiface.CallID, storiface.WorkerReturn, interface{}, *storiface.CallError) error{
-	AddPiece:              rfunc(storiface.WorkerReturn.ReturnAddPiece),
-	SealPreCommit1:        rfunc(storiface.WorkerReturn.ReturnSealPreCommit1),
-	SealPreCommit2:        rfunc(storiface.WorkerReturn.ReturnSealPreCommit2),
-	SealCommit1:           rfunc(storiface.WorkerReturn.ReturnSealCommit1),
-	SealCommit2:           rfunc(storiface.WorkerReturn.ReturnSealCommit2),
-	FinalizeSector:        rfunc(storiface.WorkerReturn.ReturnFinalizeSector),
-	ReleaseUnsealed:       rfunc(storiface.WorkerReturn.ReturnReleaseUnsealed),
-	ReplicaUpdate:         rfunc(storiface.WorkerReturn.ReturnReplicaUpdate),
-	ProveReplicaUpdate1:   rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate1),
-	ProveReplicaUpdate2:   rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate2),
-	GenerateSectorKey:     rfunc(storiface.WorkerReturn.ReturnGenerateSectorKeyFromData),
-	FinalizeReplicaUpdate: rfunc(storiface.WorkerReturn.ReturnFinalizeReplicaUpdate),
-	MoveStorage:           rfunc(storiface.WorkerReturn.ReturnMoveStorage),
-	UnsealPiece:           rfunc(storiface.WorkerReturn.ReturnUnsealPiece),
-	Fetch:                 rfunc(storiface.WorkerReturn.ReturnFetch),
+	AddPiece:            rfunc(storiface.WorkerReturn.ReturnAddPiece),
+	SealPreCommit1:      rfunc(storiface.WorkerReturn.ReturnSealPreCommit1),
+	SealPreCommit2:      rfunc(storiface.WorkerReturn.ReturnSealPreCommit2),
+	SealCommit1:         rfunc(storiface.WorkerReturn.ReturnSealCommit1),
+	SealCommit2:         rfunc(storiface.WorkerReturn.ReturnSealCommit2),
+	FinalizeSector:      rfunc(storiface.WorkerReturn.ReturnFinalizeSector),
+	ReleaseUnsealed:     rfunc(storiface.WorkerReturn.ReturnReleaseUnsealed),
+	ReplicaUpdate:       rfunc(storiface.WorkerReturn.ReturnReplicaUpdate),
+	ProveReplicaUpdate1: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate1),
+	ProveReplicaUpdate2: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate2),
+	GenerateSectorKey:   rfunc(storiface.WorkerReturn.ReturnGenerateSectorKeyFromData),
+	MoveStorage:         rfunc(storiface.WorkerReturn.ReturnMoveStorage),
+	UnsealPiece:         rfunc(storiface.WorkerReturn.ReturnUnsealPiece),
+	Fetch:               rfunc(storiface.WorkerReturn.ReturnFetch),
 }
 
 func (l *LocalWorker) asyncCall(ctx context.Context, sector storage.SectorRef, rt ReturnType, work func(ctx context.Context, ci storiface.CallID) (interface{}, error)) (storiface.CallID, error) {
@@ -458,27 +456,6 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector storage.SectorR
 	})
 }
 
-func (l *LocalWorker) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) {
-	sb, err := l.executor()
-	if err != nil {
-		return storiface.UndefCall, err
-	}
-
-	return l.asyncCall(ctx, sector, FinalizeReplicaUpdate, func(ctx context.Context, ci storiface.CallID) (interface{}, error) {
-		if err := sb.FinalizeReplicaUpdate(ctx, sector, keepUnsealed); err != nil {
-			return nil, xerrors.Errorf("finalizing sector: %w", err)
-		}
-
-		if len(keepUnsealed) == 0 {
-			if err := l.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); err != nil {
-				return nil, xerrors.Errorf("removing unsealed data: %w", err)
-			}
-		}
-
-		return nil, err
-	})
-}
-
 func (l *LocalWorker) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error) {
 	return storiface.UndefCall, xerrors.Errorf("implement me")
 }

extern/sector-storage/worker_tracked.go

@@ -215,8 +215,4 @@ func (t *trackedWorker) ProveReplicaUpdate2(ctx context.Context, sector storage.
 	})
 }
 
-func (t *trackedWorker) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) {
-	return t.tracker.track(ctx, t.execute, t.wid, t.workerInfo, sector, sealtasks.TTFinalizeReplicaUpdate, func() (storiface.CallID, error) { return t.Worker.FinalizeReplicaUpdate(ctx, sector, keepUnsealed) })
-}
-
 var _ Worker = &trackedWorker{}

extern/storage-sealing/checks.go

@@ -20,7 +20,6 @@ import (
 //  We should implement some wait-for-api logic
 type ErrApi struct{ error }
 
-type ErrNoDeals struct{ error }
 type ErrInvalidDeals struct{ error }
 type ErrInvalidPiece struct{ error }
 type ErrExpiredDeals struct{ error }
@@ -39,14 +38,12 @@ type ErrCommitWaitFailed struct{ error }
 type ErrBadRU struct{ error }
 type ErrBadPR struct{ error }
 
-func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api SealingAPI, mustHaveDeals bool) error {
+func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api SealingAPI) error {
 	tok, height, err := api.ChainHead(ctx)
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("getting chain head: %w", err)}
 	}
 
-	dealCount := 0
-
 	for i, p := range si.Pieces {
 		// if no deal is associated with the piece, ensure that we added it as
 		// filler (i.e. ensure that it has a zero PieceCID)
@@ -58,8 +55,6 @@ func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api
 			continue
 		}
 
-		dealCount++
-
 		proposal, err := api.StateMarketStorageDealProposal(ctx, p.DealInfo.DealID, tok)
 		if err != nil {
 			return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)}
@@ -82,17 +77,13 @@ func checkPieces(ctx context.Context, maddr address.Address, si SectorInfo, api
 		}
 	}
 
-	if mustHaveDeals && dealCount <= 0 {
-		return &ErrNoDeals{(xerrors.Errorf("sector %d must have deals, but does not", si.SectorNumber))}
-	}
-
 	return nil
 }
 
 // checkPrecommit checks that data commitment generated in the sealing process
 //  matches pieces, and that the seal ticket isn't expired
 func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, tok TipSetToken, height abi.ChainEpoch, api SealingAPI) (err error) {
-	if err := checkPieces(ctx, maddr, si, api, false); err != nil {
+	if err := checkPieces(ctx, maddr, si, api); err != nil {
 		return err
 	}
 
@@ -193,7 +184,7 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte,
 		return &ErrInvalidProof{xerrors.New("invalid proof (compute error?)")}
 	}
 
-	if err := checkPieces(ctx, m.maddr, si, m.Api, false); err != nil {
+	if err := checkPieces(ctx, m.maddr, si, m.Api); err != nil {
 		return err
 	}
 
@@ -203,7 +194,7 @@ func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte,
 // check that sector info is good after running a replica update
 func checkReplicaUpdate(ctx context.Context, maddr address.Address, si SectorInfo, tok TipSetToken, api SealingAPI) error {
 
-	if err := checkPieces(ctx, maddr, si, api, true); err != nil {
+	if err := checkPieces(ctx, maddr, si, api); err != nil {
 		return err
 	}
 	if !si.CCUpdate {

extern/storage-sealing/fsm.go

@@ -137,32 +137,27 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	SnapDealsWaitDeals: planOne(
 		on(SectorAddPiece{}, SnapDealsAddPiece),
 		on(SectorStartPacking{}, SnapDealsPacking),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	SnapDealsAddPiece: planOne(
 		on(SectorPieceAdded{}, SnapDealsWaitDeals),
 		apply(SectorStartPacking{}),
 		apply(SectorAddPiece{}),
 		on(SectorAddPieceFailed{}, SnapDealsAddPieceFailed),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	SnapDealsPacking: planOne(
 		on(SectorPacked{}, UpdateReplica),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	UpdateReplica: planOne(
 		on(SectorReplicaUpdate{}, ProveReplicaUpdate),
 		on(SectorUpdateReplicaFailed{}, ReplicaUpdateFailed),
 		on(SectorDealsExpired{}, SnapDealsDealsExpired),
 		on(SectorInvalidDealIDs{}, SnapDealsRecoverDealIDs),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	ProveReplicaUpdate: planOne(
 		on(SectorProveReplicaUpdate{}, SubmitReplicaUpdate),
 		on(SectorProveReplicaUpdateFailed{}, ReplicaUpdateFailed),
 		on(SectorDealsExpired{}, SnapDealsDealsExpired),
 		on(SectorInvalidDealIDs{}, SnapDealsRecoverDealIDs),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	SubmitReplicaUpdate: planOne(
 		on(SectorReplicaUpdateSubmitted{}, ReplicaUpdateWait),
@@ -174,14 +169,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	FinalizeReplicaUpdate: planOne(
-		on(SectorFinalized{}, UpdateActivating),
-	),
-	UpdateActivating: planOne(
-		on(SectorUpdateActive{}, ReleaseSectorKey),
-	),
-	ReleaseSectorKey: planOne(
-		on(SectorKeyReleased{}, Proving),
-		on(SectorReleaseKeyFailed{}, ReleaseSectorKeyFailed),
+		on(SectorFinalized{}, Proving),
 	),
 	// Sealing errors
 
@@ -243,7 +231,6 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorRetryWaitDeals{}, SnapDealsWaitDeals),
 		apply(SectorStartPacking{}),
 		apply(SectorAddPiece{}),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
 	),
 	SnapDealsDealsExpired: planOne(
 		on(SectorAbortUpgrade{}, AbortUpgrade),
@@ -262,10 +249,6 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorRetryProveReplicaUpdate{}, ProveReplicaUpdate),
 		on(SectorInvalidDealIDs{}, SnapDealsRecoverDealIDs),
 		on(SectorDealsExpired{}, SnapDealsDealsExpired),
-		on(SectorAbortUpgrade{}, AbortUpgrade),
-	),
-	ReleaseSectorKeyFailed: planOne(
-		on(SectorUpdateActive{}, ReleaseSectorKey),
 	),
 
 	// Post-seal
@@ -494,10 +477,6 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleReplicaUpdateWait, processed, nil
 	case FinalizeReplicaUpdate:
 		return m.handleFinalizeReplicaUpdate, processed, nil
-	case UpdateActivating:
-		return m.handleUpdateActivating, processed, nil
-	case ReleaseSectorKey:
-		return m.handleReleaseSectorKey, processed, nil
 
 	// Handled failure modes
 	case AddPieceFailed:
@@ -534,8 +513,6 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleSnapDealsRecoverDealIDs, processed, nil
 	case ReplicaUpdateFailed:
 		return m.handleSubmitReplicaUpdateFailed, processed, nil
-	case ReleaseSectorKeyFailed:
-		return m.handleReleaseSectorKeyFailed, 0, err
 	case AbortUpgrade:
 		return m.handleAbortUpgrade, processed, nil
 

extern/storage-sealing/fsm_events.go

@@ -335,14 +335,6 @@ type SectorReplicaUpdateLanded struct{}
 
 func (evt SectorReplicaUpdateLanded) apply(state *SectorInfo) {}
 
-type SectorUpdateActive struct{}
-
-func (evt SectorUpdateActive) apply(state *SectorInfo) {}
-
-type SectorKeyReleased struct{}
-
-func (evt SectorKeyReleased) apply(state *SectorInfo) {}
-
 // Failed state recovery
 
 type SectorRetrySealPreCommit1 struct{}
@@ -453,13 +445,6 @@ type SectorSubmitReplicaUpdateFailed struct{}
 
 func (evt SectorSubmitReplicaUpdateFailed) apply(state *SectorInfo) {}
 
-type SectorReleaseKeyFailed struct{ error }
-
-func (evt SectorReleaseKeyFailed) FormatError(xerrors.Printer) (next error) {
-	return evt.error
-}
-func (evt SectorReleaseKeyFailed) apply(state *SectorInfo) {}
-
 // Faults
 
 type SectorFaulty struct{}

extern/storage-sealing/input.go

@@ -16,7 +16,6 @@ import (
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
@@ -118,25 +117,9 @@ func (m *Sealing) maybeStartSealing(ctx statemachine.Context, sector SectorInfo,
 			return false, xerrors.Errorf("getting storage config: %w", err)
 		}
 
+		// todo check deal age, start sealing if any deal has less than X (configurable) to start deadline
 		sealTime := time.Unix(sector.CreationTime, 0).Add(cfg.WaitDealsDelay)
 
-		// check deal age, start sealing when the deal closest to starting is within slack time
-		_, current, err := m.Api.ChainHead(ctx.Context())
-		blockTime := time.Second * time.Duration(build.BlockDelaySecs)
-		if err != nil {
-			return false, xerrors.Errorf("API error getting head: %w", err)
-		}
-		for _, piece := range sector.Pieces {
-			if piece.DealInfo == nil {
-				continue
-			}
-			dealSafeSealEpoch := piece.DealInfo.DealProposal.StartEpoch - cfg.StartEpochSealingBuffer
-			dealSafeSealTime := time.Now().Add(time.Duration(dealSafeSealEpoch-current) * blockTime)
-			if dealSafeSealTime.Before(sealTime) {
-				sealTime = dealSafeSealTime
-			}
-		}
-
 		if now.After(sealTime) {
 			log.Infow("starting to seal deal sector", "sector", sector.SectorNumber, "trigger", "wait-timeout")
 			return true, ctx.Send(SectorStartPacking{})
@@ -492,10 +475,6 @@ func (m *Sealing) tryCreateDealSector(ctx context.Context, sp abi.RegisteredSeal
 		return xerrors.Errorf("getting storage config: %w", err)
 	}
 
-	if !cfg.MakeNewSectorForDeals {
-		return nil
-	}
-
 	if cfg.MaxSealingSectorsForDeals > 0 && m.stats.curSealing() >= cfg.MaxSealingSectorsForDeals {
 		return nil
 	}
@@ -545,13 +524,6 @@ func (m *Sealing) StartPacking(sid abi.SectorNumber) error {
 	return m.sectors.Send(uint64(sid), SectorStartPacking{})
 }
 
-func (m *Sealing) AbortUpgrade(sid abi.SectorNumber) error {
-	m.startupWait.Wait()
-
-	log.Infow("aborting upgrade of sector", "sector", sid, "trigger", "user")
-	return m.sectors.Send(uint64(sid), SectorAbortUpgrade{xerrors.New("triggered by user")})
-}
-
 func proposalCID(deal api.PieceDealInfo) cid.Cid {
 	pc, err := deal.DealProposal.Cid()
 	if err != nil {

extern/storage-sealing/sealiface/config.go

@@ -18,8 +18,6 @@ type Config struct {
 	// includes failed, 0 = no limit
 	MaxSealingSectorsForDeals uint64
 
-	MakeNewSectorForDeals bool
-
 	WaitDealsDelay time.Duration
 
 	CommittedCapacitySectorLifetime time.Duration

extern/storage-sealing/sector_state.go

@@ -52,14 +52,11 @@ var ExistSectorStateList = map[SectorState]struct{}{
 	ProveReplicaUpdate:      {},
 	SubmitReplicaUpdate:     {},
 	ReplicaUpdateWait:       {},
-	UpdateActivating:        {},
-	ReleaseSectorKey:        {},
 	FinalizeReplicaUpdate:   {},
 	SnapDealsAddPieceFailed: {},
 	SnapDealsDealsExpired:   {},
 	SnapDealsRecoverDealIDs: {},
 	ReplicaUpdateFailed:     {},
-	ReleaseSectorKeyFailed:  {},
 	AbortUpgrade:            {},
 }
 
@@ -107,8 +104,6 @@ const (
 	SubmitReplicaUpdate   SectorState = "SubmitReplicaUpdate"
 	ReplicaUpdateWait     SectorState = "ReplicaUpdateWait"
 	FinalizeReplicaUpdate SectorState = "FinalizeReplicaUpdate"
-	UpdateActivating      SectorState = "UpdateActivating"
-	ReleaseSectorKey      SectorState = "ReleaseSectorKey"
 
 	// error modes
 	FailedUnrecoverable  SectorState = "FailedUnrecoverable"
@@ -129,7 +124,6 @@ const (
 	SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs"
 	AbortUpgrade            SectorState = "AbortUpgrade"
 	ReplicaUpdateFailed     SectorState = "ReplicaUpdateFailed"
-	ReleaseSectorKeyFailed  SectorState = "ReleaseSectorKeyFailed"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain
@@ -159,7 +153,7 @@ func toStatState(st SectorState, finEarly bool) statSectorState {
 			return sstProving
 		}
 		return sstSealing
-	case Proving, UpdateActivating, ReleaseSectorKey, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed:
+	case Proving, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed:
 		return sstProving
 	}
 

extern/storage-sealing/states_failed.go

@@ -211,7 +211,7 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 
 	tok, _, err := m.Api.ChainHead(ctx.Context())
 	if err != nil {
-		log.Errorf("handleSubmitReplicaUpdateFailed: api error, not proceeding: %+v", err)
+		log.Errorf("handleCommitting: api error, not proceeding: %+v", err)
 		return nil
 	}
 
@@ -237,17 +237,6 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 		}
 	}
 
-	// Abort upgrade for sectors that went faulty since being marked for upgrade
-	active, err := sectorActive(ctx.Context(), m.Api, m.maddr, tok, sector.SectorNumber)
-	if err != nil {
-		log.Errorf("sector active check: api error, not proceeding: %+v", err)
-		return nil
-	}
-	if !active {
-		log.Errorf("sector marked for upgrade %d no longer active, aborting upgrade", sector.SectorNumber)
-		return ctx.Send(SectorAbortUpgrade{})
-	}
-
 	if err := failedCooldown(ctx, sector); err != nil {
 		return err
 	}
@@ -255,16 +244,6 @@ func (m *Sealing) handleSubmitReplicaUpdateFailed(ctx statemachine.Context, sect
 	return ctx.Send(SectorRetrySubmitReplicaUpdate{})
 }
 
-func (m *Sealing) handleReleaseSectorKeyFailed(ctx statemachine.Context, sector SectorInfo) error {
-	// not much we can do, wait for a bit and try again
-
-	if err := failedCooldown(ctx, sector); err != nil {
-		return err
-	}
-
-	return ctx.Send(SectorUpdateActive{})
-}
-
 func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo) error {
 	tok, _, err := m.Api.ChainHead(ctx.Context())
 	if err != nil {
@@ -499,7 +478,7 @@ func (m *Sealing) HandleRecoverDealIDs(ctx statemachine.Context, sector SectorIn
 }
 
 func (m *Sealing) handleSnapDealsRecoverDealIDs(ctx statemachine.Context, sector SectorInfo) error {
-	return m.handleRecoverDealIDsOrFailWith(ctx, sector, SectorAbortUpgrade{xerrors.New("failed recovering deal ids")})
+	return m.handleRecoverDealIDsOrFailWith(ctx, sector, SectorAbortUpgrade{})
 }
 
 func recoveryPiecesToFix(ctx context.Context, api SealingAPI, sector SectorInfo, maddr address.Address) ([]int, int, error) {

extern/storage-sealing/states_replica_update.go

@@ -2,21 +2,17 @@ package sealing
 
 import (
 	"bytes"
-	"context"
-	"time"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
 	statemachine "github.com/filecoin-project/go-statemachine"
 	api "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"golang.org/x/xerrors"
 )
 
 func (m *Sealing) handleReplicaUpdate(ctx statemachine.Context, sector SectorInfo) error {
-	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api, true); err != nil { // Sanity check state
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api); err != nil { // Sanity check state
 		return handleErrors(ctx, err, sector)
 	}
 	out, err := m.sealer.ReplicaUpdate(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber), sector.pieceInfos())
@@ -35,28 +31,13 @@ func (m *Sealing) handleProveReplicaUpdate(ctx statemachine.Context, sector Sect
 	if sector.CommR == nil {
 		return xerrors.Errorf("invalid sector %d with nil CommR", sector.SectorNumber)
 	}
-	// Abort upgrade for sectors that went faulty since being marked for upgrade
-	tok, _, err := m.Api.ChainHead(ctx.Context())
-	if err != nil {
-		log.Errorf("handleProveReplicaUpdate: api error, not proceeding: %+v", err)
-		return nil
-	}
-	active, err := sectorActive(ctx.Context(), m.Api, m.maddr, tok, sector.SectorNumber)
-	if err != nil {
-		log.Errorf("sector active check: api error, not proceeding: %+v", err)
-		return nil
-	}
-	if !active {
-		log.Errorf("sector marked for upgrade %d no longer active, aborting upgrade", sector.SectorNumber)
-		return ctx.Send(SectorAbortUpgrade{})
-	}
 
 	vanillaProofs, err := m.sealer.ProveReplicaUpdate1(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber), *sector.CommR, *sector.UpdateSealed, *sector.UpdateUnsealed)
 	if err != nil {
 		return ctx.Send(SectorProveReplicaUpdateFailed{xerrors.Errorf("prove replica update (1) failed: %w", err)})
 	}
 
-	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api, true); err != nil { // Sanity check state
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api); err != nil { // Sanity check state
 		return handleErrors(ctx, err, sector)
 	}
 
@@ -78,6 +59,10 @@ func (m *Sealing) handleSubmitReplicaUpdate(ctx statemachine.Context, sector Sec
 		return nil
 	}
 
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api); err != nil { // Sanity check state
+		return handleErrors(ctx, err, sector)
+	}
+
 	if err := checkReplicaUpdate(ctx.Context(), m.maddr, sector, tok, m.Api); err != nil {
 		return ctx.Send(SectorSubmitReplicaUpdateFailed{})
 	}
@@ -211,76 +196,16 @@ func (m *Sealing) handleReplicaUpdateWait(ctx statemachine.Context, sector Secto
 	}
 
 	if !si.SealedCID.Equals(*sector.UpdateSealed) {
-		return ctx.Send(SectorAbortUpgrade{xerrors.Errorf("mismatch of expected onchain sealed cid after replica update, expected %s got %s", sector.UpdateSealed, si.SealedCID)})
+		log.Errorf("mismatch of expected onchain sealed cid after replica update, expected %s got %s", sector.UpdateSealed, si.SealedCID)
+		return ctx.Send(SectorAbortUpgrade{})
 	}
 	return ctx.Send(SectorReplicaUpdateLanded{})
 }
 
 func (m *Sealing) handleFinalizeReplicaUpdate(ctx statemachine.Context, sector SectorInfo) error {
-	cfg, err := m.getConfig()
-	if err != nil {
-		return xerrors.Errorf("getting sealing config: %w", err)
-	}
-
-	if err := m.sealer.FinalizeReplicaUpdate(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber), sector.keepUnsealedRanges(false, cfg.AlwaysKeepUnsealedCopy)); err != nil {
-		return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("finalize sector: %w", err)})
-	}
-
 	return ctx.Send(SectorFinalized{})
 }
 
-func (m *Sealing) handleUpdateActivating(ctx statemachine.Context, sector SectorInfo) error {
-	try := func() error {
-		mw, err := m.Api.StateWaitMsg(ctx.Context(), *sector.ReplicaUpdateMessage)
-		if err != nil {
-			return err
-		}
-
-		tok, _, err := m.Api.ChainHead(ctx.Context())
-		if err != nil {
-			return err
-		}
-
-		nv, err := m.Api.StateNetworkVersion(ctx.Context(), tok)
-		if err != nil {
-			return err
-		}
-
-		lb := policy.GetWinningPoStSectorSetLookback(nv)
-
-		targetHeight := mw.Height + lb + InteractivePoRepConfidence
-
-		return m.events.ChainAt(func(context.Context, TipSetToken, abi.ChainEpoch) error {
-			return ctx.Send(SectorUpdateActive{})
-		}, func(ctx context.Context, ts TipSetToken) error {
-			log.Warn("revert in handleUpdateActivating")
-			return nil
-		}, InteractivePoRepConfidence, targetHeight)
-	}
-
-	for {
-		err := try()
-		if err == nil {
-			break
-		}
-
-		log.Errorw("error in handleUpdateActivating", "error", err)
-
-		// likely an API issue, sleep for a bit and retry
-		time.Sleep(time.Minute)
-	}
-
-	return nil
-}
-
-func (m *Sealing) handleReleaseSectorKey(ctx statemachine.Context, sector SectorInfo) error {
-	if err := m.sealer.ReleaseSectorKey(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorType, sector.SectorNumber)); err != nil {
-		return ctx.Send(SectorReleaseKeyFailed{err})
-	}
-
-	return ctx.Send(SectorKeyReleased{})
-}
-
 func handleErrors(ctx statemachine.Context, err error, sector SectorInfo) error {
 	switch err.(type) {
 	case *ErrApi:

extern/storage-sealing/states_sealing.go

@@ -198,7 +198,7 @@ func (m *Sealing) handleGetTicket(ctx statemachine.Context, sector SectorInfo) e
 }
 
 func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error {
-	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api, false); err != nil { // Sanity check state
+	if err := checkPieces(ctx.Context(), m.maddr, sector, m.Api); err != nil { // Sanity check state
 		switch err.(type) {
 		case *ErrApi:
 			log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err)

extern/storage-sealing/upgrade_queue.go

@@ -3,7 +3,6 @@ package sealing
 import (
 	"context"
 
-	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	market7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/market"
 
@@ -87,11 +86,19 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 		return xerrors.Errorf("failed to read sector on chain info: %w", err)
 	}
 
-	active, err := sectorActive(ctx, m.Api, m.maddr, tok, id)
+	active, err := m.Api.StateMinerActiveSectors(ctx, m.maddr, tok)
 	if err != nil {
-		return xerrors.Errorf("failed to check if sector is active")
+		return xerrors.Errorf("failed to check active sectors: %w", err)
 	}
-	if !active {
+	// Ensure the upgraded sector is active
+	var found bool
+	for _, si := range active {
+		if si.SectorNumber == id {
+			found = true
+			break
+		}
+	}
+	if !found {
 		return xerrors.Errorf("cannot mark inactive sector for upgrade")
 	}
 
@@ -103,22 +110,6 @@ func (m *Sealing) MarkForSnapUpgrade(ctx context.Context, id abi.SectorNumber) e
 	return m.sectors.Send(uint64(id), SectorStartCCUpdate{})
 }
 
-func sectorActive(ctx context.Context, api SealingAPI, maddr address.Address, tok TipSetToken, sector abi.SectorNumber) (bool, error) {
-	active, err := api.StateMinerActiveSectors(ctx, maddr, tok)
-	if err != nil {
-		return false, xerrors.Errorf("failed to check active sectors: %w", err)
-	}
-	// Check if sector is among active sectors
-	var found bool
-	for _, si := range active {
-		if si.SectorNumber == sector {
-			found = true
-			break
-		}
-	}
-	return found, nil
-}
-
 func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreCommitInfo) big.Int {
 	if len(params.DealIDs) == 0 {
 		return big.Zero()

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/elastic/gosigar v0.14.1
 	github.com/etclabscore/go-openrpc-reflect v0.0.36
 	github.com/fatih/color v1.13.0
-	github.com/filecoin-project/dagstore v0.4.4
+	github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3
 	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
 	github.com/filecoin-project/go-address v0.0.6
 	github.com/filecoin-project/go-bitfield v0.2.4
@@ -36,14 +36,17 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.14.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.19.0
+	github.com/filecoin-project/go-fil-markets v1.19.1-0.20220215120226-a0db7a73d4fd
+	github.com/filecoin-project/go-indexer-core v0.2.8
 	github.com/filecoin-project/go-jsonrpc v0.1.5
+	github.com/filecoin-project/go-legs v0.3.0
 	github.com/filecoin-project/go-padreader v0.0.1
-	github.com/filecoin-project/go-paramfetch v0.0.4
+	github.com/filecoin-project/go-paramfetch v0.0.3-0.20220111000201-e42866db1a53
 	github.com/filecoin-project/go-state-types v0.1.3
 	github.com/filecoin-project/go-statemachine v1.0.1
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
+	github.com/filecoin-project/index-provider v0.3.0
 	github.com/filecoin-project/specs-actors v0.9.14
 	github.com/filecoin-project/specs-actors/v2 v2.3.6
 	github.com/filecoin-project/specs-actors/v3 v3.1.1
@@ -51,7 +54,7 @@ require (
 	github.com/filecoin-project/specs-actors/v5 v5.0.4
 	github.com/filecoin-project/specs-actors/v6 v6.0.1
 	github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1
-	github.com/filecoin-project/specs-storage v0.2.0
+	github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9
 	github.com/filecoin-project/test-vectors/schema v0.0.5
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
 	github.com/gdamore/tcell/v2 v2.2.0
@@ -77,7 +80,7 @@ require (
 	github.com/ipfs/go-ds-badger2 v0.1.2
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ds-measure v0.2.0
-	github.com/ipfs/go-fs-lock v0.0.6
+	github.com/ipfs/go-fs-lock v0.0.7
 	github.com/ipfs/go-graphsync v0.12.0
 	github.com/ipfs/go-ipfs-blockstore v1.1.2
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
@@ -96,9 +99,9 @@ require (
 	github.com/ipfs/go-merkledag v0.5.1
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-path v0.0.7
+	github.com/ipfs/go-path v0.2.1
 	github.com/ipfs/go-unixfs v0.3.1
-	github.com/ipfs/interface-go-ipfs-core v0.4.0
+	github.com/ipfs/interface-go-ipfs-core v0.5.2
 	github.com/ipld/go-car v0.3.3
 	github.com/ipld/go-car/v2 v2.1.1
 	github.com/ipld/go-codec-dagpb v1.3.0
@@ -108,7 +111,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-eventbus v0.2.1
-	github.com/libp2p/go-libp2p v0.18.0-rc4
+	github.com/libp2p/go-libp2p v0.18.0-rc3
 	github.com/libp2p/go-libp2p-connmgr v0.3.1 // indirect
 	github.com/libp2p/go-libp2p-core v0.14.0
 	github.com/libp2p/go-libp2p-discovery v0.6.0
@@ -116,13 +119,13 @@ require (
 	github.com/libp2p/go-libp2p-noise v0.3.0
 	github.com/libp2p/go-libp2p-peerstore v0.6.0
 	github.com/libp2p/go-libp2p-pubsub v0.6.1
-	github.com/libp2p/go-libp2p-quic-transport v0.16.1
+	github.com/libp2p/go-libp2p-quic-transport v0.16.0
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/libp2p/go-libp2p-resource-manager v0.1.3
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3
 	github.com/libp2p/go-libp2p-swarm v0.10.1
 	github.com/libp2p/go-libp2p-tls v0.3.1
-	github.com/libp2p/go-libp2p-yamux v0.8.2
+	github.com/libp2p/go-libp2p-yamux v0.8.1
 	github.com/libp2p/go-maddr-filter v0.1.0
 	github.com/mattn/go-isatty v0.0.14
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
@@ -143,9 +146,9 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
-	github.com/urfave/cli/v2 v2.2.0
+	github.com/urfave/cli/v2 v2.3.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8
+	github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325
@@ -155,11 +158,9 @@ require (
 	go.opentelemetry.io/otel/bridge/opencensus v0.25.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.2.0
 	go.opentelemetry.io/otel/sdk v1.2.0
-	go.uber.org/dig v1.10.0 // indirect
-	go.uber.org/fx v1.9.0
+	go.uber.org/fx v1.15.0
 	go.uber.org/multierr v1.7.0
 	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211209171907-798191bca915
@@ -168,9 +169,10 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools v2.2.0+incompatible
-	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
+
+replace github.com/ipfs/go-path => github.com/ipfs/go-path v0.0.7

go.sum

@@ -1,3 +1,4 @@
+bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -87,6 +88,7 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o=
 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -96,6 +98,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -129,6 +132,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
 github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
@@ -159,6 +163,7 @@ github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRt
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -166,6 +171,7 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
@@ -198,6 +204,7 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -269,6 +276,7 @@ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6
 github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/gosigar v0.14.1 h1:T0aQ7n/n2ZA9W7DmAnj60v+qzqKERdBgJBO1CG2W6rc=
 github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE=
 github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -290,9 +298,9 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/filecoin-project/dagstore v0.4.3-0.20211211192320-72b849e131d2/go.mod h1:tlV8C11UljvFq3WWlMh2oMViEaVaPb6uT8eL/YQgDfk=
-github.com/filecoin-project/dagstore v0.4.4 h1:luolWahhzp3ulRsapGKE7raoLE3n2cFkQUJjPyqUmF4=
-github.com/filecoin-project/dagstore v0.4.4/go.mod h1:7BlOvaTJrFJ1Qemt5jHlLJ4VhDIuSIzGS0IwO/0AXPA=
+github.com/filecoin-project/dagstore v0.5.1/go.mod h1:OdlK3x5m3Mol874WC2bI79H4H2+leN+FabwWdW2D/wY=
+github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3 h1:kuZsRFDjyrDAgoHNolJBnO7xX1EETHdJ090sgD3Al5E=
+github.com/filecoin-project/dagstore v0.5.2-0.20220120115845-e07b050f48d3/go.mod h1:OdlK3x5m3Mol874WC2bI79H4H2+leN+FabwWdW2D/wY=
 github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
 github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
 github.com/filecoin-project/go-address v0.0.6 h1:DWQtj38ax+ogHwyH3VULRIoT8E6loyXqsk/p81xoY7M=
@@ -316,6 +324,7 @@ github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9AN
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o=
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
+github.com/filecoin-project/go-dagaggregator-unixfs v0.2.0/go.mod h1:WTuJWgBQY0omnQqa8kRPT9O0Uj5wQOgslVMUuTeHdJ8=
 github.com/filecoin-project/go-data-transfer v1.14.0 h1:4pnfJk8FYtqcdAg+QRGzaz57seUC/Tz+HJgPuGB7zdg=
 github.com/filecoin-project/go-data-transfer v1.14.0/go.mod h1:wNJKhaLLYBJDM3VFvgvYi4iUjPa69pz/1Q5Q4HzX2wE=
 github.com/filecoin-project/go-ds-versioning v0.0.0-20211206185234-508abd7c2aff/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4=
@@ -327,8 +336,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88Oq
 github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
-github.com/filecoin-project/go-fil-markets v1.19.0 h1:kap2q2wTM6tfkVO5gMA5DD9GUeTvkDhMfhjCtEwMDM8=
-github.com/filecoin-project/go-fil-markets v1.19.0/go.mod h1:qsb3apmo4RSJYCEq40QxVdU7UZospN6nFJLOBHuaIbc=
+github.com/filecoin-project/go-fil-markets v1.19.1-0.20220215120226-a0db7a73d4fd h1:RO4M6fD9L6+OGqgk6OnMrFJcN5JkcMo3MX40vqxUfmQ=
+github.com/filecoin-project/go-fil-markets v1.19.1-0.20220215120226-a0db7a73d4fd/go.mod h1:Q7EWzO9s4sg2KYBUJduTYdtVhVhusDuH2W0keyyub3I=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -336,19 +345,23 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+
 github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI=
 github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI=
 github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g=
+github.com/filecoin-project/go-indexer-core v0.2.4/go.mod h1:MSe5aRWmfRB+5syR4yDV+OApgJU+MUJ4rl9VUuzwCfc=
+github.com/filecoin-project/go-indexer-core v0.2.8 h1:h1SRdZKTVcaXlzex3UevHh4OWDAhgpMzL4tHNAA7MQI=
+github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43PjRCn3yBuUiZgIxuA0hTUamY=
 github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk=
 github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
+github.com/filecoin-project/go-legs v0.2.7/go.mod h1:NrdELuDbtAH8/xqRMgyOYms67aliQajExInLS6g8zFM=
+github.com/filecoin-project/go-legs v0.3.0 h1:1rDNdPdXbgetmmvRcYZV5YIplIO8LtBVQ7ZttKCrTrM=
+github.com/filecoin-project/go-legs v0.3.0/go.mod h1:x6nwM+DuN7NzlPndOoJuiHYCX+pze6+efPRx17nIA7M=
 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak=
-github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
 github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs=
 github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
-github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8=
-github.com/filecoin-project/go-paramfetch v0.0.4/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts=
+github.com/filecoin-project/go-paramfetch v0.0.3-0.20220111000201-e42866db1a53 h1:+nripp+UI/rhl01w9Gs4V0XDGaVPYPMGU/D/gNVLue0=
+github.com/filecoin-project/go-paramfetch v0.0.3-0.20220111000201-e42866db1a53/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts=
 github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I=
 github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
-github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0=
@@ -361,6 +374,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd
 github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo=
 github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus=
 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
+github.com/filecoin-project/index-provider v0.3.0 h1:+oU4whsKq2x2Z/sQfd2gVGDrARrOZVpR/irlGsKaxSc=
+github.com/filecoin-project/index-provider v0.3.0/go.mod h1:3m4D2O+o649LfrsgNCZqTiiY6r+t9qF5g55jhB3PPLo=
 github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
 github.com/filecoin-project/specs-actors v0.9.14 h1:68PVstg2UB3ZsMLF+DKFTAs/YKsqhKWynkr0IqmVRQY=
 github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
@@ -382,11 +397,14 @@ github.com/filecoin-project/specs-actors/v7 v7.0.0-20211117170924-fd07a4c7dff9/g
 github.com/filecoin-project/specs-actors/v7 v7.0.0-20211222192039-c83bea50c402/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE=
 github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1 h1:FuDaXIbcw2hRsFI8SDTmsGGCE+NumpF6aiBoU/2X5W4=
 github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M=
-github.com/filecoin-project/specs-storage v0.2.0 h1:Y4UDv0apRQ3zI2GiPPubi8JblpUZZphEdaJUxCutfyg=
-github.com/filecoin-project/specs-storage v0.2.0/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU=
+github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9 h1:oUYOvF7EvdXS0Zmk9mNkaB6Bu0l+WXBYPzVodKMiLug=
+github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU=
+github.com/filecoin-project/storetheindex v0.3.0 h1:M9C4BFLCKn6FJn2N/8q0VbX00hEjEWUuvJpscqKwqzQ=
+github.com/filecoin-project/storetheindex v0.3.0/go.mod h1:GhUmP8+nINxDRUqxQvTc3Fqc7Yp6w0dW/uTaYYIzd68=
 github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg=
 github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
 github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -399,8 +417,14 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
 github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
 github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
+github.com/gammazero/keymutex v0.0.2 h1:cmpLBJHdEwn+WlR5Z/o9/BN92znSZTp5AKPQDpu1QcI=
+github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs=
+github.com/gammazero/radixtree v0.2.5 h1:muPQ4eEgCkUymFWPiVQRuXOQv4IhWg8YXH2r71MoqPM=
+github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0=
 github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4=
 github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
@@ -409,6 +433,7 @@ github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U
 github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -665,6 +690,8 @@ github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIc
 github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis=
 github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
 github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
+github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
+github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI=
 github.com/ipfs/go-bitswap v0.5.1 h1:721YAEDBnLIrvcIMkCHCdqp34hA8jwL9yKMkyJpSpco=
 github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo=
 github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
@@ -673,6 +700,8 @@ github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/d
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
 github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So=
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
+github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
+github.com/ipfs/go-blockservice v0.1.7/go.mod h1:GmS+BAt4hrwBKkzE11AFDQUrnvqjwFatGS2MY7wOjEM=
 github.com/ipfs/go-blockservice v0.2.1 h1:NJ4j/cwEfIg60rzAWcCIxRtOwbf6ZPK49MewNxObCPQ=
 github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -695,6 +724,7 @@ github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRV
 github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs=
 github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
@@ -712,6 +742,7 @@ github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1
 github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek=
 github.com/ipfs/go-ds-badger2 v0.1.2 h1:sQc2q1gaXrv8YFNeUtxil0neuyDf9hnVHfLsi7lpXfE=
 github.com/ipfs/go-ds-badger2 v0.1.2/go.mod h1:3FtQmDv6fMubygEfU43bsFelYpIiXX/XEYA54l9eCwg=
+github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
 github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
 github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
@@ -720,15 +751,25 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
 github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
 github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
+github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE=
+github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo=
+github.com/ipfs/go-filestore v0.1.0/go.mod h1:0KTrzoJnJ3sJDEDM09Vq8nz8H475rRyeq4i0n/bpF00=
 github.com/ipfs/go-filestore v1.1.0 h1:Pu4tLBi1bucu6/HU9llaOmb9yLFk/sgP+pW764zNDoE=
 github.com/ipfs/go-filestore v1.1.0/go.mod h1:6e1/5Y6NvLuCRdmda/KA4GUhXJQ3Uat6vcWm2DJfxc8=
-github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0=
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
+github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
+github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
+github.com/ipfs/go-graphsync v0.11.0/go.mod h1:wC+c8vGVjAHthsVIl8LKr37cUra2GOaMYcQNNmMxDqE=
 github.com/ipfs/go-graphsync v0.12.0 h1:QCsVHVzb9FTkcm3NEa8GjXnUeGit1L9s08HcSVQ4m/g=
 github.com/ipfs/go-graphsync v0.12.0/go.mod h1:nASYWYETgsnMbQ3+DirNImOHQ8TY0a5AhAqyOY55tUg=
+github.com/ipfs/go-ipfs v0.11.0/go.mod h1:g68Thu2Ho11AWoHsN34P5fSK7iA6OWWRy3T/g8HLixc=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
+github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
+github.com/ipfs/go-ipfs-blockstore v0.1.6/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
 github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
+github.com/ipfs/go-ipfs-blockstore v1.0.4-0.20210205083733-fb07d7bc5aec/go.mod h1:feuklK+m9POeWJzYQO7l05yNEgUiX5oELBNA8/Be33E=
+github.com/ipfs/go-ipfs-blockstore v1.0.4/go.mod h1:uL7/gTJ8QIZ3MtA3dWf+s1a0U3fJy2fcEZAsovpRp+w=
 github.com/ipfs/go-ipfs-blockstore v1.1.1/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
 github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw=
 github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
@@ -737,15 +778,18 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
-github.com/ipfs/go-ipfs-cmds v0.3.0 h1:mi9oYrSCox5aBhutqAYqw6/9crlyGbw4E/aJtwS4zI4=
 github.com/ipfs/go-ipfs-cmds v0.3.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
-github.com/ipfs/go-ipfs-config v0.5.3 h1:3GpI/xR9FoJNTjU6YvCMRbYyEi0dBVY5UtlUTcNRlSA=
+github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw=
+github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
 github.com/ipfs/go-ipfs-config v0.5.3/go.mod h1:nSLCFtlaL+2rbl3F+9D4gQZQbT1LjRKx7TJg/IHz6oM=
+github.com/ipfs/go-ipfs-config v0.18.0 h1:Ta1aNGNEq6RIvzbw7dqzCVZJKb7j+Dd35JFnAOCpT8g=
+github.com/ipfs/go-ipfs-config v0.18.0/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
 github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
+github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
 github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
 github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU=
 github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
@@ -761,11 +805,14 @@ github.com/ipfs/go-ipfs-files v0.0.9 h1:OFyOfmuVDu9c5YtjSDORmwXzE6fmZikzZpzsnNkg
 github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84=
 github.com/ipfs/go-ipfs-http-client v0.0.6 h1:k2QllZyP7Fz5hMgsX5hvHfn1WPG9Ngdy5WknQ7JNhBM=
 github.com/ipfs/go-ipfs-http-client v0.0.6/go.mod h1:8e2dQbntMZKxLfny+tyXJ7bJHZFERp/2vyzZdvkeLMc=
+github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
+github.com/ipfs/go-ipfs-pinner v0.2.1/go.mod h1:l1AtLL5bovb7opnG77sh4Y10waINz3Y1ni6CvTzx7oo=
 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
 github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
 github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
 github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY=
 github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
+github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8=
 github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
 github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
 github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY=
@@ -784,6 +831,7 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA=
 github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
+github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI=
 github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI=
 github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc=
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
@@ -814,39 +862,49 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
+github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUOg3ojR5MOE=
 github.com/ipfs/go-merkledag v0.5.1 h1:tr17GPP5XtPhvPPiWtu20tSGZiZDuTaJRXBLcr79Umk=
 github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
+github.com/ipfs/go-mfs v0.2.1/go.mod h1:Woj80iuw4ajDnIP6+seRaoHpPsc9hmL0pk/nDNDWP88=
+github.com/ipfs/go-namesys v0.4.0/go.mod h1:jpJwzodyP8DZdWN6DShRjVZw6gaqMr4nQLBSxU5cR6E=
 github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
 github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ=
 github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY=
 github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
 github.com/ipfs/go-peertaskqueue v0.7.1 h1:7PLjon3RZwRQMgOTvYccZ+mjzkmds/7YzSWKFlBAypE=
 github.com/ipfs/go-peertaskqueue v0.7.1/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
+github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
 github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4=
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
+github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8=
 github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
+github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=
 github.com/ipfs/go-unixfsnode v1.2.0 h1:tHHBJftsJyHGa8bS62PpkYNqHy/Sug3c/vxxC8NaGQY=
 github.com/ipfs/go-unixfsnode v1.2.0/go.mod h1:mQEgLjxkV/1mohkC4p7taRRBYPBeXu97SA3YaerT2q0=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
-github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg=
 github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
+github.com/ipfs/interface-go-ipfs-core v0.5.2 h1:m1/5U+WpOK2ZE7Qzs5iIu80QM1ZA3aWYi2Ilwpi+tdg=
+github.com/ipfs/interface-go-ipfs-core v0.5.2/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE=
 github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo=
 github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg=
 github.com/ipfs/iptb-plugins v0.3.0 h1:C1rpq1o5lUZtaAOkLIox5akh6ba4uk/3RwWc6ttVxw0=
 github.com/ipfs/iptb-plugins v0.3.0/go.mod h1:5QtOvckeIw4bY86gSH4fgh3p3gCSMn3FmIKr4gaBncA=
+github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM=
 github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g=
+github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE=
 github.com/ipld/go-car v0.3.3-0.20211210032800-e6f244225a16/go.mod h1:/wkKF4908ULT4dFIFIUZYcfjAnj+KFnJvlh8Hsz1FbQ=
 github.com/ipld/go-car v0.3.3 h1:D6y+jvg9h2ZSv7GLUMWUwg5VTLy1E7Ak+uQw5orOg3I=
 github.com/ipld/go-car v0.3.3/go.mod h1:/wkKF4908ULT4dFIFIUZYcfjAnj+KFnJvlh8Hsz1FbQ=
-github.com/ipld/go-car/v2 v2.1.1-0.20211211000942-be2525f6bf2d/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
 github.com/ipld/go-car/v2 v2.1.1 h1:saaKz4nC0AdfCGHLYKeXLGn8ivoPC54fyS55uyOLKwA=
 github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
 github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
@@ -857,7 +915,10 @@ github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/j
 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
+github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
 github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM=
+github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
+github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc=
 github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
@@ -866,6 +927,9 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
+github.com/ipld/go-storethehash v0.0.0-20210915160027-d72ca9b0968c/go.mod h1:PwE6iq8TiWJRI3zMGA1RtkFAnrDMK93dLA5SUeu0lH8=
+github.com/ipld/go-storethehash v0.0.1 h1:U9E9n3dE0IaImSNbRcUmm5Dk9UPiS9H0i+LmCFFGJBI=
+github.com/ipld/go-storethehash v0.0.1/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
@@ -973,6 +1037,7 @@ github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5
 github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70=
 github.com/libp2p/go-conn-security-multistream v0.3.0 h1:9UCIKlBL1hC9u7nkMXpD1nkc/T53PKMAn3/k9ivBAVc=
 github.com/libp2p/go-conn-security-multistream v0.3.0/go.mod h1:EEP47t4fw/bTelVmEzIDqSe69hO/ip52xBEhZMLWAHM=
+github.com/libp2p/go-doh-resolver v0.3.1/go.mod h1:y5go1ZppAq9N2eppbX0xON01CyPBeUg2yS6BTssssog=
 github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
 github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
 github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc=
@@ -990,13 +1055,16 @@ github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZk
 github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k=
 github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw=
 github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o=
+github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0=
+github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo=
+github.com/libp2p/go-libp2p v0.14.0/go.mod h1:dsQrWLAoIn+GkHPN/U+yypizkHiB9tnv79Os+kSgQ4Q=
 github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0=
 github.com/libp2p/go-libp2p v0.14.4/go.mod h1:EIRU0Of4J5S8rkockZM7eJp2S0UrCyi55m2kJVru3rM=
 github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4=
 github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw=
 github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8=
-github.com/libp2p/go-libp2p v0.18.0-rc4 h1:OUsSbeu7q+Ck/bV9wHDxFzb08ORqBupHhpCmRBhWrJ8=
-github.com/libp2p/go-libp2p v0.18.0-rc4/go.mod h1:wzmsk1ioOq9FGQys2BN5BIw4nugP6+R+CyW3JbPEbbs=
+github.com/libp2p/go-libp2p v0.18.0-rc3 h1:tI+dAFDgOCeHRF6FgvXpqbrVz+ZFabX/pXO2BUdHu4o=
+github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M=
 github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
 github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E=
 github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I=
@@ -1006,6 +1074,7 @@ github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/
 github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI=
 github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI=
 github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A=
+github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk=
 github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk=
 github.com/libp2p/go-libp2p-autonat v0.6.0/go.mod h1:bFC6kY8jwzNNWoqc8iGE57vsfwyJ/lP4O4DOV1e0B2o=
 github.com/libp2p/go-libp2p-autonat v0.7.0/go.mod h1:uPvPn6J7cN+LCfFwW5tpOYvAz5NvPTc4iBamTV/WDMg=
@@ -1078,8 +1147,12 @@ github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQO
 github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug=
 github.com/libp2p/go-libp2p-discovery v0.6.0 h1:1XdPmhMJr8Tmj/yUfkJMIi8mgwWrLUsCB3bMxdT+DSo=
 github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8=
+github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc=
+github.com/libp2p/go-libp2p-gostream v0.3.1 h1:XlwohsPn6uopGluEWs1Csv1QCEjrTXf2ZQagzZ5paAg=
+github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI=
 github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go=
 github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8=
+github.com/libp2p/go-libp2p-http v0.2.1/go.mod h1:9KdioZ7XqNH0eZkZG9bulZLzHv11A7/12fT97agqWhg=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
@@ -1100,6 +1173,7 @@ github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3
 github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
 github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo=
 github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek=
+github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs=
 github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw=
 github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g=
 github.com/libp2p/go-libp2p-mplex v0.5.0 h1:vt3k4E4HSND9XH4Z8rUpacPJFSAgLOv6HDvG8W9Ks9E=
@@ -1114,6 +1188,7 @@ github.com/libp2p/go-libp2p-net v0.0.2/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8
 github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q=
 github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ=
 github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
+github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM=
 github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q=
 github.com/libp2p/go-libp2p-noise v0.3.0 h1:NCVH7evhVt9njbTQshzT7N1S3Q6fjj9M11FCgfH5+cA=
 github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ=
@@ -1142,15 +1217,15 @@ github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uz
 github.com/libp2p/go-libp2p-pubsub v0.6.0/go.mod h1:nJv87QM2cU0w45KPR1rZicq+FmFIOD16zmT+ep1nOmg=
 github.com/libp2p/go-libp2p-pubsub v0.6.1 h1:wycbV+f4rreCoVY61Do6g/BUk0RIrbNRcYVbn+QkjGk=
 github.com/libp2p/go-libp2p-pubsub v0.6.1/go.mod h1:nJv87QM2cU0w45KPR1rZicq+FmFIOD16zmT+ep1nOmg=
+github.com/libp2p/go-libp2p-pubsub-router v0.5.0/go.mod h1:TRJKskSem3C0aSb3CmRgPwq6IleVFzds6hS09fmZbGM=
 github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU=
 github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA=
 github.com/libp2p/go-libp2p-quic-transport v0.11.2/go.mod h1:wlanzKtIh6pHrq+0U3p3DY9PJfGqxMgPaGKaK5LifwQ=
 github.com/libp2p/go-libp2p-quic-transport v0.13.0/go.mod h1:39/ZWJ1TW/jx1iFkKzzUg00W6tDJh73FC0xYudjr7Hc=
 github.com/libp2p/go-libp2p-quic-transport v0.15.0/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ=
 github.com/libp2p/go-libp2p-quic-transport v0.15.2/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ=
+github.com/libp2p/go-libp2p-quic-transport v0.16.0 h1:aVg9/jr+R2esov5sH7wkXrmYmqJiUjtLMLYX3L9KYdY=
 github.com/libp2p/go-libp2p-quic-transport v0.16.0/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ=
-github.com/libp2p/go-libp2p-quic-transport v0.16.1 h1:N/XqYXHurphPLDfXYhll8NyqzdZYQqAF4GIr7+SmLV8=
-github.com/libp2p/go-libp2p-quic-transport v0.16.1/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ=
 github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q=
 github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
 github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg=
@@ -1176,6 +1251,8 @@ github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaT
 github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM=
 github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM=
 github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
+github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
+github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw=
 github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4=
 github.com/libp2p/go-libp2p-swarm v0.5.3/go.mod h1:NBn7eNW2lu568L7Ns9wdFrOhgRlkRnIDg0FLKbuu3i8=
 github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc=
@@ -1207,6 +1284,7 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2
 github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
 github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns=
 github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o=
+github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.3/go.mod h1:bpkldbOWXMrXhpZbSV1mQxTrefOg2Fi+k1ClDSA4ppw=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.6/go.mod h1:JE0WQuQdy+uLZ5zOaI3Nw9dWGYJIA7mywEtP2lMvnyk=
@@ -1226,13 +1304,14 @@ github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhL
 github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4=
 github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30=
 github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po=
+github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4=
+github.com/libp2p/go-libp2p-yamux v0.5.3/go.mod h1:Vy3TMonBAfTMXHWopsMc8iX/XGRYrRlpUaMzaeuHV/s=
 github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE=
 github.com/libp2p/go-libp2p-yamux v0.6.0/go.mod h1:MRhd6mAYnFRnSISp4M8i0ClV/j+mWHo2mYLifWGw33k=
 github.com/libp2p/go-libp2p-yamux v0.7.0/go.mod h1:fMyA0CsPfHkIuBU0wjRGrCjTBFiXTXxG0k5M4ETv+08=
 github.com/libp2p/go-libp2p-yamux v0.8.0/go.mod h1:yTkPgN2ib8FHyU1ZcVD7aelzyAqXXwEPbyx+aSKm9h8=
+github.com/libp2p/go-libp2p-yamux v0.8.1 h1:pi7zUeZ4Z9TpbUMntvSvoP3dFD4SEw/VPybxBcOZGzg=
 github.com/libp2p/go-libp2p-yamux v0.8.1/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE=
-github.com/libp2p/go-libp2p-yamux v0.8.2 h1:6GKWntresp0TFxMP/oSoH96nV8XKJRdynXsdp43dn0Y=
-github.com/libp2p/go-libp2p-yamux v0.8.2/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE=
 github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
 github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
 github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M=
@@ -1283,6 +1362,7 @@ github.com/libp2p/go-reuseport-transport v0.1.0/go.mod h1:vev0C0uMkzriDY59yFHD9v
 github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
+github.com/libp2p/go-socket-activation v0.1.0/go.mod h1:gzda2dNkMG5Ti2OfWNNwW0FDIbj0g/aJJU320FcLfhk=
 github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
 github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ=
 github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw=
@@ -1294,6 +1374,7 @@ github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19
 github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
 github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
 github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0=
+github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M=
 github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU=
 github.com/libp2p/go-tcp-transport v0.2.4/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU=
 github.com/libp2p/go-tcp-transport v0.2.7/go.mod h1:lue9p1b3VmZj1MhhEGB/etmvF/nBQ0X9CW2DutBT3MM=
@@ -1307,6 +1388,7 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw
 github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y=
 github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM=
 github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
+github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
 github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA=
 github.com/libp2p/go-ws-transport v0.5.0/go.mod h1:I2juo1dNTbl8BKSBYo98XY85kU2xds1iamArLvl8kNg=
 github.com/libp2p/go-ws-transport v0.6.0 h1:326XBL6Q+5CQ2KtjXz32+eGu02W/Kz2+Fm4SpXdr0q4=
@@ -1321,6 +1403,8 @@ github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h
 github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
 github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI=
 github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
+github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U=
+github.com/libp2p/go-yamux/v2 v2.1.1/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ=
 github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ=
 github.com/libp2p/go-yamux/v2 v2.3.0/go.mod h1:iTU+lOIn/2h0AgKcL49clNTwfEw+WSfDYrXe05EyKIs=
 github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo=
@@ -1385,6 +1469,7 @@ github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
@@ -1403,6 +1488,7 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -1470,6 +1556,7 @@ github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/94
 github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.1.0/go.mod h1:01k2RAqtoXIuPa3DCavAE9/6jc6nM0H3EgZyfUhN2oY=
 github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
+github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
 github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
@@ -1489,8 +1576,9 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw
 github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
-github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c h1:VyANTtZ0wsx0IAZnCZhfMmAmfUyzJq/5JQi2hHOtKS0=
 github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
+github.com/multiformats/go-multicodec v0.4.0 h1:fbqb6ky7erjdD+/zaEBJgZWu1i8D6i/wmPywGK7sdow=
+github.com/multiformats/go-multicodec v0.4.0/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
@@ -1499,12 +1587,14 @@ github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpK
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
+github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag=
 github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA=
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38=
+github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k=
 github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k=
 github.com/multiformats/go-multistream v0.2.2 h1:TCYu1BHTDr1F/Qm75qwYISQdzGcRdC21nFgQW7l7GBo=
 github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs=
@@ -1787,6 +1877,7 @@ github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
@@ -1800,8 +1891,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
@@ -1809,8 +1901,9 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
-github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA=
 github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
+github.com/warpfork/go-testmark v0.9.0 h1:nc+uaCiv5lFQLYjhuC2LTYeJ7JaC+gdDmsz9r0ISy0Y=
+github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w=
@@ -1838,8 +1931,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8 h1:TEv7MId88TyIqIUL4hbf9otOookIolMxlEbN0ro671Y=
-github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008 h1:7WtW9D9VGpmRLuQmrPy2JobUNdka95z3MKEVpELtOjo=
+github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g=
@@ -1848,6 +1941,7 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
 github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
+github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso=
 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 h1:NwiwjQDB3CzQ5XH0rdMh1oQqzJH7O2PSLWxif/w3zsY=
 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4/go.mod h1:K+EVq8d5QcQ2At5VECsA+SNZvWefyBXh8TnIsxo1OvQ=
 github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
@@ -1869,6 +1963,7 @@ github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/
 github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8=
 github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1941,14 +2036,15 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
-go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
-go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
-go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
+go.uber.org/dig v1.12.0 h1:l1GQeZpEbss0/M4l/ZotuBndCrkMdjnygzgcuOjAdaY=
+go.uber.org/dig v1.12.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
+go.uber.org/fx v1.15.0 h1:kcfBpAm98n0ksanyyZLFE/Q3T7yPi13Ge2liu3TxR+A=
+go.uber.org/fx v1.15.0/go.mod h1:jI3RazQUhGv5KkpZIRv+kuP4CcgX3fnc0qX8bLnzbx8=
 go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
@@ -1998,6 +2094,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -2013,6 +2110,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2119,6 +2218,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -2198,6 +2298,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191206220618-eeba5f6aabab/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -2247,7 +2348,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915 h1:P+8mCzuEpyszAT6T42q0sxU+eveBAF/cJ2Kp0x6/8+0=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -2472,6 +2575,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
 gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -2521,6 +2625,7 @@ modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc=
 modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

itests/ccupgrade_test.go

@@ -30,14 +30,23 @@ func TestCCUpgrade(t *testing.T) {
 	//stm: @MINER_SECTOR_LIST_001
 	kit.QuietMiningLogs()
 
-	runTestCCUpgrade(t)
+	for _, height := range []abi.ChainEpoch{
+		-1,  // before
+		162, // while sealing
+		560, // after upgrade deal
+	} {
+		height := height // make linters happy by copying
+		t.Run(fmt.Sprintf("upgrade-%d", height), func(t *testing.T) {
+			runTestCCUpgrade(t, height)
+		})
+	}
 }
 
-func runTestCCUpgrade(t *testing.T) *kit.TestFullNode {
+func runTestCCUpgrade(t *testing.T, upgradeHeight abi.ChainEpoch) *kit.TestFullNode {
 	ctx := context.Background()
 	blockTime := 1 * time.Millisecond
 
-	client, miner, ens := kit.EnsembleMinimal(t, kit.GenesisNetworkVersion(network.Version15), kit.ThroughRPC())
+	client, miner, ens := kit.EnsembleMinimal(t, kit.GenesisNetworkVersion(network.Version15))
 	ens.InterconnectAll().BeginMiningMustPost(blockTime)
 
 	maddr, err := miner.ActorAddress(ctx)
@@ -63,6 +72,7 @@ func runTestCCUpgrade(t *testing.T) *kit.TestFullNode {
 	}
 	waitForSectorActive(ctx, t, CCUpgrade, client, maddr)
 
+	//stm: @SECTOR_CC_UPGRADE_001
 	err = miner.SectorMarkForUpgrade(ctx, sl[0], true)
 	require.NoError(t, err)
 
@@ -81,11 +91,6 @@ func runTestCCUpgrade(t *testing.T) *kit.TestFullNode {
 	status, err := miner.SectorsStatus(ctx, CCUpgrade, true)
 	require.NoError(t, err)
 	assert.Equal(t, 1, len(status.Deals))
-
-	miner.WaitSectorsProving(ctx, map[abi.SectorNumber]struct{}{
-		CCUpgrade: {},
-	})
-
 	return client
 }
 
@@ -132,7 +137,7 @@ func TestCCUpgradeAndPoSt(t *testing.T) {
 	kit.QuietMiningLogs()
 	t.Run("upgrade and then post", func(t *testing.T) {
 		ctx := context.Background()
-		n := runTestCCUpgrade(t)
+		n := runTestCCUpgrade(t, 100)
 		ts, err := n.ChainHead(ctx)
 		require.NoError(t, err)
 		start := ts.Height()

itests/deals_anycid_test.go

@@ -0,0 +1,162 @@
+package itests
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipld/go-car"
+	"github.com/ipld/go-car/v2/blockstore"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/modules"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+func TestDealRetrieveByAnyCid(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
+
+	ctx := context.Background()
+	oldDelay := policy.GetPreCommitChallengeDelay()
+	policy.SetPreCommitChallengeDelay(5)
+	t.Cleanup(func() {
+		policy.SetPreCommitChallengeDelay(oldDelay)
+	})
+
+	// Allow 8MB sectors
+	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1)
+	kit.QuietMiningLogs()
+
+	// For these tests where the block time is artificially short, just use
+	// a deal start epoch that is guaranteed to be far enough in the future
+	// so that the deal starts sealing in time
+	startEpoch := abi.ChainEpoch(2 << 12)
+
+	// Override the dependency injection for the blockstore accessor, so that
+	// we can get a reference to the blockstore containing our deal later in
+	// the test
+	var bsa storagemarket.BlockstoreAccessor
+	bsaFn := func(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor {
+		bsa = modules.StorageBlockstoreAccessor(importmgr)
+		return bsa
+	}
+	bsaOpt := kit.ConstructorOpts(node.Override(new(storagemarket.BlockstoreAccessor), bsaFn))
+
+	// Allow 8MB sectors
+	eightMBSectorsOpt := kit.SectorSize(8 << 20)
+
+	// Create a client, and a miner with its own full node
+	_, client, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(), bsaOpt, eightMBSectorsOpt)
+	ens.InterconnectAll().BeginMining(250 * time.Millisecond)
+
+	dh := kit.NewDealHarness(t, client, miner, miner)
+
+	// Generate a DAG with multiple levels, so that we can test the case where
+	// the client requests a CID for a block which is not the root block but
+	// does have a subtree below it in the DAG
+	dagOpts := kit.GeneratedDAGOpts{
+		// Max size of a block
+		ChunkSize: 1024,
+		// Max links from a block to other blocks
+		Maxlinks: 10,
+	}
+	carv1FilePath, _ := kit.CreateRandomCARv1(t, 5, 100*1024, dagOpts)
+	res, err := client.ClientImport(ctx, api.FileRef{Path: carv1FilePath, IsCAR: true})
+	require.NoError(t, err)
+
+	// Get the blockstore for the file
+	bs, err := bsa.Get(res.Root)
+	require.NoError(t, err)
+
+	// Get all CIDs from the file
+	sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: res.Root, Selector: selectorparse.CommonSelector_ExploreAllRecursively}})
+	prepared, err := sc.Prepare()
+	require.NoError(t, err)
+	cids := prepared.Cids()
+	for i, c := range cids {
+		blk, err := bs.Get(ctx, c)
+		require.NoError(t, err)
+
+		nd, err := ipld.Decode(blk)
+		require.NoError(t, err)
+
+		t.Log(i, c, len(nd.Links()))
+	}
+
+	// Create a storage deal
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = res.Root
+	dp.DealStartEpoch = startEpoch
+	dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price
+	dealCid := dh.StartDeal(ctx, dp)
+
+	// Wait for the deal to be sealed
+	dh.WaitDealSealed(ctx, dealCid, false, false, nil)
+
+	ask, err := miner.MarketGetRetrievalAsk(ctx)
+	require.NoError(t, err)
+	ask.PricePerByte = abi.NewTokenAmount(0)
+	ask.UnsealPrice = abi.NewTokenAmount(0)
+	err = miner.MarketSetRetrievalAsk(ctx, ask)
+	require.NoError(t, err)
+
+	// Fetch the deal data
+	info, err := client.ClientGetDealInfo(ctx, *dealCid)
+	require.NoError(t, err)
+
+	// Make retrievals against CIDs at different levels in the DAG
+	cidIndices := []int{1, 11, 27, 32, 47}
+	for _, val := range cidIndices {
+		t.Logf("performing retrieval for cid at index %d", val)
+
+		targetCid := cids[val]
+		offer, err := client.ClientMinerQueryOffer(ctx, miner.ActorAddr, targetCid, &info.PieceCID)
+		require.NoError(t, err)
+		require.Empty(t, offer.Err)
+
+		// retrieve in a CAR file and ensure roots match
+		outputCar := dh.PerformRetrieval(ctx, dealCid, targetCid, true, offer)
+		_, err = os.Stat(outputCar)
+		require.NoError(t, err)
+		f, err := os.Open(outputCar)
+		require.NoError(t, err)
+		ch, err := car.ReadHeader(bufio.NewReader(f))
+		require.NoError(t, err)
+		require.EqualValues(t, ch.Roots[0], targetCid)
+		require.NoError(t, f.Close())
+
+		// create CAR from original file starting at targetCid and ensure it matches the retrieved CAR file.
+		tmp, err := os.CreateTemp(t.TempDir(), "randcarv1")
+		require.NoError(t, err)
+		rd, err := blockstore.OpenReadOnly(carv1FilePath, blockstore.UseWholeCIDs(true))
+		require.NoError(t, err)
+		err = car.NewSelectiveCar(
+			ctx,
+			rd,
+			[]car.Dag{{
+				Root:     targetCid,
+				Selector: selectorparse.CommonSelector_ExploreAllRecursively,
+			}},
+		).Write(tmp)
+		require.NoError(t, err)
+		require.NoError(t, tmp.Close())
+		require.NoError(t, rd.Close())
+
+		kit.AssertFilesEqual(t, tmp.Name(), outputCar)
+		t.Log("car files match")
+	}
+}

itests/deals_concurrent_test.go

@@ -8,13 +8,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/stretchr/testify/require"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/shared_testutil"
 	"github.com/filecoin-project/go-state-types/abi"
+	provider "github.com/filecoin-project/index-provider"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
@@ -47,7 +49,9 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
 	runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
 		api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me
 
-		client, main, market, _ := kit.EnsembleWithMinerAndMarketNodes(t, kit.ThroughRPC())
+		idxProv := shared_testutil.NewMockIndexProvider()
+		idxProvOpt := kit.ConstructorOpts(node.Override(new(provider.Interface), idxProv))
+		client, main, market, _ := kit.EnsembleWithMinerAndMarketNodes(t, kit.ThroughRPC(), idxProvOpt)
 
 		dh := kit.NewDealHarness(t, client, main, market)
 
@@ -56,6 +60,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
 			FastRetrieval: fastRetrieval,
 			CarExport:     carExport,
 			StartEpoch:    startEpoch,
+			IndexProvider: idxProv,
 		})
 	}
 

itests/kit/client.go

@@ -111,9 +111,14 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode *TestFullNode)
 	tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-client")
 	require.NoError(t, err)
 	path := filepath.Join(tmpdir, "outfile.dat")
-	out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path)
-	fmt.Println("retrieve:\n", out)
-	require.Regexp(t, regexp.MustCompile("Success"), out)
+
+	for {
+		out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path)
+		fmt.Println("retrieve:\n", out)
+		if strings.Contains(out, "Success") {
+			break
+		}
+	}
 }
 
 func CreateImportFile(ctx context.Context, client api.FullNode, rseed int, size int) (res *api.ImportRes, path string, data []byte, err error) {

itests/kit/deals.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/shared_testutil"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
@@ -305,14 +306,20 @@ func (dh *DealHarness) StartSealingWaiting(ctx context.Context) {
 	}
 }
 
-func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool) (path string) {
-	// perform retrieval.
-	info, err := dh.client.ClientGetDealInfo(ctx, *deal)
-	require.NoError(dh.t, err)
+func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool, offers ...api.QueryOffer) (path string) {
+	var offer api.QueryOffer
+	if len(offers) == 0 {
+		// perform retrieval.
+		info, err := dh.client.ClientGetDealInfo(ctx, *deal)
+		require.NoError(dh.t, err)
 
-	offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID)
-	require.NoError(dh.t, err)
-	require.NotEmpty(dh.t, offers, "no offers")
+		offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID)
+		require.NoError(dh.t, err)
+		require.NotEmpty(dh.t, offers, "no offers")
+		offer = offers[0]
+	} else {
+		offer = offers[0]
+	}
 
 	carFile, err := ioutil.TempFile(dh.t.TempDir(), "ret-car")
 	require.NoError(dh.t, err)
@@ -326,7 +333,7 @@ func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root
 	updates, err := dh.client.ClientGetRetrievalUpdates(updatesCtx)
 	require.NoError(dh.t, err)
 
-	retrievalRes, err := dh.client.ClientRetrieve(ctx, offers[0].Order(caddr))
+	retrievalRes, err := dh.client.ClientRetrieve(ctx, offer.Order(caddr))
 	require.NoError(dh.t, err)
 consumeEvents:
 	for {
@@ -363,11 +370,6 @@ consumeEvents:
 		}))
 
 	ret := carFile.Name()
-	if carExport {
-		actualFile := dh.ExtractFileFromCAR(ctx, carFile)
-		ret = actualFile.Name()
-		_ = actualFile.Close() //nolint:errcheck
-	}
 
 	return ret
 }
@@ -404,9 +406,11 @@ type RunConcurrentDealsOpts struct {
 	CarExport                bool
 	StartEpoch               abi.ChainEpoch
 	UseCARFileForStorageDeal bool
+	IndexProvider            *shared_testutil.MockIndexProvider
 }
 
 func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
+	ctx := context.Background()
 	errgrp, _ := errgroup.WithContext(context.Background())
 	for i := 0; i < opts.N; i++ {
 		i := i
@@ -429,10 +433,28 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
 				UseCARFileForStorageDeal: opts.UseCARFileForStorageDeal,
 			})
 
+			// Check that the storage provider announced the deal to indexers
+			if opts.IndexProvider != nil {
+				notifs := opts.IndexProvider.GetNotifs()
+				_, ok := notifs[string(deal.Bytes())]
+				require.True(dh.t, ok)
+			}
+
 			dh.t.Logf("retrieving deal %d/%d", i, opts.N)
 
 			outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, opts.CarExport)
-			AssertFilesEqual(dh.t, inPath, outPath)
+
+			if opts.CarExport {
+				f, err := os.Open(outPath)
+				require.NoError(dh.t, err)
+				actualFile := dh.ExtractFileFromCAR(ctx, f)
+				require.NoError(dh.t, f.Close())
+
+				AssertFilesEqual(dh.t, inPath, actualFile.Name())
+			} else {
+				AssertFilesEqual(dh.t, inPath, outPath)
+			}
+
 			return nil
 		})
 	}

itests/kit/deals_state.go

@@ -14,7 +14,7 @@ func CategorizeDealState(dealStatus string) TestDealState {
 	switch dealStatus {
 	case "StorageDealFailing", "StorageDealError":
 		return TestDealStateFailed
-	case "StorageDealStaged", "StorageDealAwaitingPreCommit", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed":
+	case "StorageDealAwaitingPreCommit", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed":
 		return TestDealStateComplete
 	}
 	return TestDealStateInProgress

itests/kit/ensemble.go

@@ -36,6 +36,8 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/sector-storage/mock"
 	"github.com/filecoin-project/lotus/genesis"
+	"github.com/filecoin-project/lotus/markets/idxprov"
+	idxprov_test "github.com/filecoin-project/lotus/markets/idxprov/idxprov_test"
 	lotusminer "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/config"
@@ -541,6 +543,12 @@ func (n *Ensemble) Start() *Ensemble {
 			node.Override(new(stmgr.UpgradeSchedule), n.options.upgradeSchedule),
 		}
 
+		if m.options.subsystems.Has(SMarkets) {
+			opts = append(opts,
+				node.Override(new(idxprov.MeshCreator), idxprov_test.NewNoopMeshCreator),
+			)
+		}
+
 		// append any node builder options.
 		opts = append(opts, m.options.extraNodeOpts...)
 

itests/kit/files.go

@@ -29,7 +29,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const unixfsChunkSize uint64 = 1 << 10
+const unixfsChunkSize int64 = 1 << 10
 
 var defaultHashFunction = uint64(mh.BLAKE2B_MIN + 31)
 
@@ -54,7 +54,7 @@ func CreateRandomFile(t *testing.T, rseed, size int) (path string) {
 
 // CreateRandomFile creates a  normal file with the provided seed and the
 // provided size and then transforms it to a CARv1 file and returns it.
-func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, origFilePath string) {
+func CreateRandomCARv1(t *testing.T, rseed, size int, opts ...GeneratedDAGOpts) (carV1FilePath string, origFilePath string) {
 	ctx := context.Background()
 	if size == 0 {
 		size = 1600
@@ -75,7 +75,7 @@ func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, ori
 	bs := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
 	dagSvc := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
 
-	root := writeUnixfsDAG(ctx, t, file, dagSvc)
+	root := writeUnixfsDAG(ctx, t, file, dagSvc, opts...)
 
 	// create a CARv1 file from the DAG
 	tmp, err := os.CreateTemp(t.TempDir(), "randcarv1")
@@ -92,7 +92,20 @@ func CreateRandomCARv1(t *testing.T, rseed, size int) (carV1FilePath string, ori
 	return tmp.Name(), file.Name()
 }
 
-func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldformat.DAGService) cid.Cid {
+type GeneratedDAGOpts struct {
+	ChunkSize int64
+	Maxlinks  int
+}
+
+func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldformat.DAGService, opts ...GeneratedDAGOpts) cid.Cid {
+	dagOpts := GeneratedDAGOpts{
+		ChunkSize: unixfsChunkSize,
+		Maxlinks:  1024,
+	}
+	if len(opts) > 0 {
+		dagOpts = opts[0]
+	}
+
 	rpf := files.NewReaderFile(rd)
 
 	// generate the dag and get the root
@@ -103,7 +116,7 @@ func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldfor
 
 	bufferedDS := ipldformat.NewBufferedDAG(ctx, dag)
 	params := ihelper.DagBuilderParams{
-		Maxlinks:  1024,
+		Maxlinks:  dagOpts.Maxlinks,
 		RawLeaves: true,
 		CidBuilder: cidutil.InlineBuilder{
 			Builder: prefix,
@@ -112,7 +125,7 @@ func writeUnixfsDAG(ctx context.Context, t *testing.T, rd io.Reader, dag ipldfor
 		Dagserv: bufferedDS,
 	}
 
-	db, err := params.New(chunk.NewSizeSplitter(rpf, int64(unixfsChunkSize)))
+	db, err := params.New(chunk.NewSizeSplitter(rpf, dagOpts.ChunkSize))
 	require.NoError(t, err)
 
 	nd, err := balanced.Layout(db)

itests/kit/node_miner.go

@@ -87,10 +87,7 @@ type TestMiner struct {
 
 func (tm *TestMiner) PledgeSectors(ctx context.Context, n, existing int, blockNotif <-chan struct{}) {
 	toCheck := tm.StartPledge(ctx, n, existing, blockNotif)
-	tm.WaitSectorsProving(ctx, toCheck)
-}
 
-func (tm *TestMiner) WaitSectorsProving(ctx context.Context, toCheck map[abi.SectorNumber]struct{}) {
 	for len(toCheck) > 0 {
 		tm.FlushSealingBatches(ctx)
 
@@ -108,8 +105,9 @@ func (tm *TestMiner) WaitSectorsProving(ctx context.Context, toCheck map[abi.Sec
 		}
 
 		build.Clock.Sleep(100 * time.Millisecond)
-		fmt.Printf("WaitSectorsProving: %d %+v\n", len(toCheck), states)
+		fmt.Printf("WaitSeal: %d %+v\n", len(toCheck), states)
 	}
+
 }
 
 func (tm *TestMiner) StartPledge(ctx context.Context, n, existing int, blockNotif <-chan struct{}) map[abi.SectorNumber]struct{} {

itests/kit/rpc.go

@@ -39,7 +39,6 @@ func fullRpc(t *testing.T, f *TestFullNode) *TestFullNode {
 	require.NoError(t, err)
 
 	srv, maddr := CreateRPCServer(t, handler, l)
-	fmt.Printf("FULLNODE RPC ENV FOR CLI DEBUGGING `export FULLNODE_API_INFO=%s`\n", "ws://"+srv.Listener.Addr().String())
 
 	cl, stop, err := client.NewFullNodeRPCV1(context.Background(), "ws://"+srv.Listener.Addr().String()+"/rpc/v1", nil)
 	require.NoError(t, err)
@@ -55,9 +54,7 @@ func minerRpc(t *testing.T, m *TestMiner) *TestMiner {
 
 	srv, maddr := CreateRPCServer(t, handler, m.RemoteListener)
 
-	fmt.Printf("creating RPC server for %s at %s\n", m.ActorAddr, srv.Listener.Addr().String())
-	fmt.Printf("SP RPC ENV FOR CLI DEBUGGING `export MINER_API_INFO=%s`\n", "ws://"+srv.Listener.Addr().String())
-
+	fmt.Println("creating RPC server for", m.ActorAddr, "at: ", srv.Listener.Addr().String())
 	url := "ws://" + srv.Listener.Addr().String() + "/rpc/v0"
 	cl, stop, err := client.NewStorageMinerRPCV0(context.Background(), url, nil)
 	require.NoError(t, err)

markets/dagstore/wrapper.go

@@ -3,12 +3,18 @@ package dagstore
 import (
 	"context"
 	"errors"
+	"fmt"
 	"math"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
+	"github.com/filecoin-project/go-indexer-core/store/storethehash"
+	"github.com/libp2p/go-libp2p-core/host"
+
+	carindex "github.com/ipld/go-car/v2/index"
+
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	levelds "github.com/ipfs/go-ds-leveldb"
@@ -53,7 +59,7 @@ type Wrapper struct {
 
 var _ stores.DAGStoreWrapper = (*Wrapper)(nil)
 
-func NewDAGStore(cfg config.DAGStoreConfig, minerApi MinerAPI) (*dagstore.DAGStore, *Wrapper, error) {
+func NewDAGStore(cfg config.DAGStoreConfig, minerApi MinerAPI, h host.Host) (*dagstore.DAGStore, *Wrapper, error) {
 	// construct the DAG Store.
 	registry := mount.NewRegistry()
 	if err := registry.Register(lotusScheme, mountTemplate(minerApi)); err != nil {
@@ -79,9 +85,15 @@ func NewDAGStore(cfg config.DAGStoreConfig, minerApi MinerAPI) (*dagstore.DAGSto
 
 	irepo, err := index.NewFSRepo(indexDir)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("failed to initialise dagstore index repo")
+		return nil, nil, xerrors.Errorf("failed to initialise dagstore index repo: %w", err)
 	}
 
+	store, err := storethehash.New(indexDir)
+	if err != nil {
+		return nil, nil, xerrors.Errorf("failed to initialise store the index: %w", err)
+	}
+	topIndex := index.NewInverted(store, h.ID())
+
 	dcfg := dagstore.Config{
 		TransientsDir: transientsDir,
 		IndexRepo:     irepo,
@@ -89,6 +101,7 @@ func NewDAGStore(cfg config.DAGStoreConfig, minerApi MinerAPI) (*dagstore.DAGSto
 		MountRegistry: registry,
 		FailureCh:     failureCh,
 		TraceCh:       traceCh,
+		TopLevelIndex: topIndex,
 		// not limiting fetches globally, as the Lotus mount does
 		// conditional throttling.
 		MaxConcurrentIndex:        cfg.MaxConcurrentIndex,
@@ -398,6 +411,33 @@ func (w *Wrapper) markRegistrationComplete() error {
 	return file.Close()
 }
 
+// Get all the pieces that contain a block
+func (w *Wrapper) GetPiecesContainingBlock(blockCID cid.Cid) ([]cid.Cid, error) {
+	// Pieces are stored as "shards" in the DAG store
+	shardKeys, err := w.dagst.ShardsContainingMultihash(blockCID.Hash())
+	if err != nil {
+		return nil, xerrors.Errorf("getting pieces containing block %s: %w", blockCID, err)
+	}
+
+	// Convert from shard key to cid
+	pieceCids := make([]cid.Cid, 0, len(shardKeys))
+	for _, k := range shardKeys {
+		c, err := cid.Parse(k.String())
+		if err != nil {
+			prefix := fmt.Sprintf("getting pieces containing block %s:", blockCID)
+			return nil, xerrors.Errorf(prefix+" converting shard key %s to piece cid: %w", k, err)
+		}
+
+		pieceCids = append(pieceCids, c)
+	}
+
+	return pieceCids, nil
+}
+
+func (w *Wrapper) GetIterableIndexForPiece(pieceCid cid.Cid) (carindex.IterableIndex, error) {
+	return w.dagst.GetIterableIndex(shard.KeyFromCID(pieceCid))
+}
+
 func (w *Wrapper) Close() error {
 	// Cancel the context
 	w.cancel()

markets/dagstore/wrapper_migration_test.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"testing"
 
+	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/dagstore"
@@ -96,8 +98,11 @@ func TestShardRegistration(t *testing.T) {
 	cfg := config.DefaultStorageMiner().DAGStore
 	cfg.RootDir = t.TempDir()
 
+	h, err := mocknet.New().GenPeer()
+	require.NoError(t, err)
+
 	mapi := NewMinerAPI(ps, &wrappedSA{sa}, 10, 5)
-	dagst, w, err := NewDAGStore(cfg, mapi)
+	dagst, w, err := NewDAGStore(cfg, mapi, h)
 	require.NoError(t, err)
 	require.NotNil(t, dagst)
 	require.NotNil(t, w)

markets/dagstore/wrapper_test.go

@@ -7,6 +7,12 @@ import (
 	"testing"
 	"time"
 
+	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
+
+	mh "github.com/multiformats/go-multihash"
+
+	carindex "github.com/ipld/go-car/v2/index"
+
 	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/xerrors"
@@ -25,11 +31,13 @@ func TestWrapperAcquireRecovery(t *testing.T) {
 	pieceCid, err := cid.Parse("bafkqaaa")
 	require.NoError(t, err)
 
+	h, err := mocknet.New().GenPeer()
+	require.NoError(t, err)
 	// Create a DAG store wrapper
 	dagst, w, err := NewDAGStore(config.DAGStoreConfig{
 		RootDir:    t.TempDir(),
 		GCInterval: config.Duration(1 * time.Millisecond),
-	}, mockLotusMount{})
+	}, mockLotusMount{}, h)
 	require.NoError(t, err)
 
 	defer dagst.Close() //nolint:errcheck
@@ -75,12 +83,14 @@ func TestWrapperAcquireRecovery(t *testing.T) {
 // TestWrapperBackground verifies the behaviour of the background go routine
 func TestWrapperBackground(t *testing.T) {
 	ctx := context.Background()
+	h, err := mocknet.New().GenPeer()
+	require.NoError(t, err)
 
 	// Create a DAG store wrapper
 	dagst, w, err := NewDAGStore(config.DAGStoreConfig{
 		RootDir:    t.TempDir(),
 		GCInterval: config.Duration(1 * time.Millisecond),
-	}, mockLotusMount{})
+	}, mockLotusMount{}, h)
 	require.NoError(t, err)
 
 	defer dagst.Close() //nolint:errcheck
@@ -130,6 +140,18 @@ type mockDagStore struct {
 	close   chan struct{}
 }
 
+func (m *mockDagStore) GetIterableIndex(key shard.Key) (carindex.IterableIndex, error) {
+	return nil, nil
+}
+
+func (m *mockDagStore) ShardsContainingMultihash(h mh.Multihash) ([]shard.Key, error) {
+	return nil, nil
+}
+
+func (m *mockDagStore) GetShardKeysForCid(c cid.Cid) ([]shard.Key, error) {
+	panic("implement me")
+}
+
 func (m *mockDagStore) DestroyShard(ctx context.Context, key shard.Key, out chan dagstore.ShardResult, _ dagstore.DestroyOpts) error {
 	panic("implement me")
 }

markets/idxprov/host.go

@@ -0,0 +1,5 @@
+package idxprov
+
+import "github.com/libp2p/go-libp2p-core/host"
+
+type Host host.Host

markets/idxprov/idxprov_test/noop.go

@@ -0,0 +1,16 @@
+package idxprov_test
+
+import (
+	"context"
+)
+
+type NoopMeshCreator struct {
+}
+
+func NewNoopMeshCreator() *NoopMeshCreator {
+	return &NoopMeshCreator{}
+}
+
+func (mc NoopMeshCreator) Connect(ctx context.Context) error {
+	return nil
+}

markets/idxprov/mesh.go

@@ -0,0 +1,54 @@
+package idxprov
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/libp2p/go-libp2p-core/peer"
+
+	"github.com/filecoin-project/lotus/api/v1api"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var log = logging.Logger("idxprov")
+
+type MeshCreator interface {
+	Connect(ctx context.Context) error
+}
+
+type Libp2pMeshCreator struct {
+	fullnodeApi v1api.FullNode
+	idxProvHost Host
+}
+
+func (mc Libp2pMeshCreator) Connect(ctx context.Context) error {
+
+	// Add the index provider's host ID to list of protected peers first, before any attempt to
+	// connect to full node.
+	idxProvID := mc.idxProvHost.ID()
+	if err := mc.fullnodeApi.NetProtectAdd(ctx, []peer.ID{idxProvID}); err != nil {
+		return fmt.Errorf("failed to call NetProtectAdd on the full node, err: %w", err)
+	}
+
+	faddrs, err := mc.fullnodeApi.NetAddrsListen(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to fetch full node listen addrs, err: %w", err)
+	}
+
+	// Connect to the full node, ask it to protect the connection and protect the connection on
+	// markets end too.
+	if err := mc.idxProvHost.Connect(ctx, faddrs); err != nil {
+		return fmt.Errorf("failed to connect index provider host with the full node: %w", err)
+	}
+	mc.idxProvHost.ConnManager().Protect(faddrs.ID, "index-provider-gossipsub")
+
+	log.Debugw("successfully connected to full node and asked it protect indexer provider peer conn", "fullNodeInfo", faddrs.String(),
+		"idxProviderPeerId", idxProvID)
+
+	return nil
+}
+
+func NewMeshCreator(fullnodeApi v1api.FullNode, idxProvHost Host) MeshCreator {
+	return Libp2pMeshCreator{fullnodeApi, idxProvHost}
+}

metrics/metrics.go

@@ -76,6 +76,8 @@ var (
 	ChainNodeHeight                     = stats.Int64("chain/node_height", "Current Height of the node", stats.UnitDimensionless)
 	ChainNodeHeightExpected             = stats.Int64("chain/node_height_expected", "Expected Height of the node", stats.UnitDimensionless)
 	ChainNodeWorkerHeight               = stats.Int64("chain/node_worker_height", "Current Height of workers on the node", stats.UnitDimensionless)
+	IndexerMessageValidationFailure     = stats.Int64("indexer/failure", "Counter for indexer message validation failures", stats.UnitDimensionless)
+	IndexerMessageValidationSuccess     = stats.Int64("indexer/success", "Counter for indexer message validation successes", stats.UnitDimensionless)
 	MessagePublished                    = stats.Int64("message/published", "Counter for total locally published messages", stats.UnitDimensionless)
 	MessageReceived                     = stats.Int64("message/received", "Counter for total received messages", stats.UnitDimensionless)
 	MessageValidationFailure            = stats.Int64("message/failure", "Counter for message validation failures", stats.UnitDimensionless)
@@ -200,6 +202,15 @@ var (
 			return view.Distribution(bounds...)
 		}(),
 	}
+	IndexerMessageValidationFailureView = &view.View{
+		Measure:     IndexerMessageValidationFailure,
+		Aggregation: view.Count(),
+		TagKeys:     []tag.Key{FailureType, Local},
+	}
+	IndexerMessageValidationSuccessView = &view.View{
+		Measure:     IndexerMessageValidationSuccess,
+		Aggregation: view.Count(),
+	}
 	MessagePublishedView = &view.View{
 		Measure:     MessagePublished,
 		Aggregation: view.Count(),
@@ -532,6 +543,8 @@ var ChainNodeViews = append([]*view.View{
 	BlockValidationSuccessView,
 	BlockValidationDurationView,
 	BlockDelayView,
+	IndexerMessageValidationFailureView,
+	IndexerMessageValidationSuccessView,
 	MessagePublishedView,
 	MessageReceivedView,
 	MessageValidationFailureView,

node/builder.go

@@ -105,6 +105,8 @@ const (
 	HandleMigrateClientFundsKey
 	HandlePaymentChannelManagerKey
 
+	RelayIndexerMessagesKey
+
 	// miner
 	GetParamsKey
 	HandleMigrateProviderFundsKey

node/builder_chain.go

@@ -134,6 +134,8 @@ var ChainNode = Options(
 
 	Override(new(*full.GasPriceCache), full.NewGasPriceCache),
 
+	Override(RelayIndexerMessagesKey, modules.RelayIndexerMessages),
+
 	// Lite node API
 	ApplyIf(isLiteNode,
 		Override(new(messagepool.Provider), messagepool.NewProviderLite),

node/builder_miner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask"
 	"github.com/filecoin-project/go-state-types/abi"
+	provider "github.com/filecoin-project/index-provider"
 	storage2 "github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
@@ -25,6 +26,7 @@ import (
 	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/filecoin-project/lotus/markets/dagstore"
 	"github.com/filecoin-project/lotus/markets/dealfilter"
+	"github.com/filecoin-project/lotus/markets/idxprov"
 	"github.com/filecoin-project/lotus/markets/retrievaladapter"
 	"github.com/filecoin-project/lotus/markets/sectoraccessor"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
@@ -167,6 +169,9 @@ func ConfigStorageMiner(c interface{}) Option {
 			Override(new(dtypes.ProviderTransferNetwork), modules.NewProviderTransferNetwork),
 			Override(new(dtypes.ProviderTransport), modules.NewProviderTransport),
 			Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDataTransfer),
+			Override(new(idxprov.MeshCreator), idxprov.NewMeshCreator),
+			Override(new(idxprov.Host), modules.IndexProviderHost(cfg.IndexProvider)),
+			Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)),
 			Override(new(*storedask.StoredAsk), modules.NewStorageAsk),
 			Override(new(dtypes.StorageDealFilter), modules.BasicDealFilter(cfg.Dealmaking, nil)),
 			Override(new(storagemarket.StorageProvider), modules.StorageProvider),

node/config/def.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+	ipconfig "github.com/filecoin-project/index-provider/config"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -85,7 +86,7 @@ func DefaultFullNode() *FullNode {
 			Splitstore: Splitstore{
 				ColdStoreType: "universal",
 				HotStoreType:  "badger",
-				MarkSetType:   "badger",
+				MarkSetType:   "map",
 
 				HotStoreFullGCFrequency: 20,
 			},
@@ -156,7 +157,6 @@ func DefaultStorageMiner() *StorageMiner {
 			ConsiderVerifiedStorageDeals:   true,
 			ConsiderUnverifiedStorageDeals: true,
 			PieceCidBlocklist:              []cid.Cid{},
-			MakeNewSectorForDeals:          true,
 			// TODO: It'd be nice to set this based on sector size
 			MaxDealStartDelay:               Duration(time.Hour * 24 * 14),
 			ExpectedSealDuration:            Duration(time.Hour * 24),
@@ -181,6 +181,16 @@ func DefaultStorageMiner() *StorageMiner {
 			},
 		},
 
+		IndexProvider: IndexProviderConfig{
+			ListenAddresses: []string{
+				"/ip4/0.0.0.0/tcp/0",
+				"/ip6/::/tcp/0",
+			},
+			AnnounceAddresses: []string{},
+
+			MaxSimultaneousTransfers: DefaultSimultaneousTransfers,
+		},
+
 		Subsystems: MinerSubsystemConfig{
 			EnableMining:        true,
 			EnableSealing:       true,
@@ -221,6 +231,11 @@ func DefaultStorageMiner() *StorageMiner {
 			GCInterval:                 Duration(1 * time.Minute),
 		},
 	}
+
+	// TODO: Remove hardcoded defaults once provider library exposes them.
+	// See: https://github.com/filecoin-project/index-provider/issues/108
+	cfg.IndexProvider.Ingest = ipconfig.NewIngest()
+
 	cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
 	cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345"
 	return cfg

node/config/def_test.go

@@ -52,3 +52,8 @@ func TestDefaultMinerRoundtrip(t *testing.T) {
 
 	require.True(t, reflect.DeepEqual(c, c2))
 }
+
+func TestDefaultStorageMiner_SetsIndexIngestTopic(t *testing.T) {
+	subject := DefaultStorageMiner()
+	require.Equal(t, "/indexer/ingest/mainnet", subject.IndexProvider.PubSubTopic)
+}