Merge branch 'master' into feat/cid-to-piece-idx

avoid panic

.circleci/config.yml

@@ -785,6 +785,11 @@ workflows:
           suite: itest-deals_512mb
           target: "./itests/deals_512mb_test.go"
       
+      - test:
+          name: test-itest-deals_anycid
+          suite: itest-deals_anycid
+          target: "./itests/deals_anycid_test.go"
+      
       - test:
           name: test-itest-deals_concurrent
           suite: itest-deals_concurrent
@@ -850,6 +855,11 @@ workflows:
           suite: itest-get_messages_in_ts
           target: "./itests/get_messages_in_ts_test.go"
       
+      - test:
+          name: test-itest-mempool
+          suite: itest-mempool
+          target: "./itests/mempool_test.go"
+      
       - test:
           name: test-itest-multisig
           suite: itest-multisig

CHANGELOG.md

@@ -1,5 +1,106 @@
 # Lotus changelog
 
+# 1.14.2 / 2022-02-24
+
+This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 
+
+Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
+
+Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out! 
+
+## Bug Fixes
+- fix lotus-bench for sealing jobs (#8173)
+- fix:sealing:really-do-it flag for abort upgrade (#8181)
+- fix:proving:post check sector handles snap deals replica faults (#8177)
+- fix: sealing: missing file type (#8180)
+
+## Others
+- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong 
+  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore, 
+  we want to retract it and users may use v1.14.1&^.
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @zenground0 | 2 | +73/-58 | 12 |
+| @eben.xie | 1 | +7/-0 | 1 |
+| @jennijuju | 1 | +4/-0 | 1 |
+| @jennijuju | 1 | +2/-1 | 1 |
+| @ribasushi | 1 | +2/-0 | 1 |
+
+# 1.14.1 / 2022-02-18
+
+This is an **optional** release of lotus, that fixes the incorrect *comment* of network v15 OhSnap upgrade **date**. Note the actual upgrade epoch in [v1.14.0](https://github.com/filecoin-project/lotus/releases/tag/v1.14.0) was correct.
+
+# 1.14.0 / 2022-02-17
+
+This is a MANDATORY release of Lotus that introduces [Filecoin network v15, 
+codenamed the OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
+
+The network is scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, and worker(s).
+
+The OhSnap upgrade introduces the following FIPs, delivered in [actors v7](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0):
+- [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
+- [FIP-0028 Remove Datacap from Verified clients](https://github.com/filecoin-project/FIPs/pull/226)
+
+It is recommended that storage providers download the new params before updating their node, miner, and workers. To do so:
+
+- Download Lotus v1.14.0 or later
+- run `make lotus-shed`
+- run `./lotus-shed fetch-params` with the appropriate `proving-params` flag
+- Upgrade the Lotus daemon and miner **when the previous step is complete**
+
+All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
+  
+## New Features and Changes
+- Integrate actor v7-rc1:
+  - Integrate v7 actors ([#7617](https://github.com/filecoin-project/lotus/pull/7617))
+  - feat: state: Fast migration for v15 ([#7933](https://github.com/filecoin-project/lotus/pull/7933))
+  - fix: blockstore: Add missing locks to autobatch::Get() [#7939](https://github.com/filecoin-project/lotus/pull/7939))
+  - correctness fixes for the autobatch blockstore ([#7940](https://github.com/filecoin-project/lotus/pull/7940))
+- Implement and support [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
+  - chore: deps: Integrate proof v11.0.0 ([#7923](https://github.com/filecoin-project/lotus/pull/7923))
+  - Snap Deals Lotus Integration: FSM Posting and integration test ([#7810](https://github.com/filecoin-project/lotus/pull/7810))
+  - Feat/sector storage unseal ([#7730](https://github.com/filecoin-project/lotus/pull/7730))
+  - Feat/snap deals storage ([#7615](https://github.com/filecoin-project/lotus/pull/7615))
+  - fix: sealing: Add more deal expiration checks during PRU pipeline ([#7871](https://github.com/filecoin-project/lotus/pull/7871))
+  - chore: deps: Update go-paramfetch ([#7917](https://github.com/filecoin-project/lotus/pull/7917))
+  - feat: #7880 gas: add gas charge for VerifyReplicaUpdate ([#7897](https://github.com/filecoin-project/lotus/pull/7897))
+  - enhancement: sectors: disable existing cc upgrade path 2 days before the upgrade epoch ([#7900](https://github.com/filecoin-project/lotus/pull/7900))
+
+## Improvements
+- updating to new datastore/blockstore code with contexts ([#7646](https://github.com/filecoin-project/lotus/pull/7646))
+- reorder transfer checks so as to ensure sending 2B FIL to yourself fails if you don't have that amount ([#7637](https://github.com/filecoin-project/lotus/pull/7637))
+- VM: Circ supply should be constant per epoch ([#7811](https://github.com/filecoin-project/lotus/pull/7811))
+
+## Bug Fixes
+- Fix: state: circsuypply calc around null blocks ([#7890](https://github.com/filecoin-project/lotus/pull/7890))
+- Mempool msg selection should respect block message limits ([#7321](https://github.com/filecoin-project/lotus/pull/7321))
+  SplitStore: supress compaction near upgrades ([#7734](https://github.com/filecoin-project/lotus/pull/7734))
+  
+## Others
+- chore: create pull_request_template.md ([#7726](https://github.com/filecoin-project/lotus/pull/7726))
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Aayush Rajasekaran | 41 | +5538/-1205 | 189 |
+| zenground0 | 11 | +3316/-524 | 124 |
+| Jennifer Wang | 29 | +714/-599 | 68 |
+| ZenGround0 | 3 | +263/-25 | 11 |
+| c r | 2 | +198/-30 | 6 |
+| vyzo | 4 | +189/-7 | 7 |
+| Aayush | 11 | +146/-48 | 49 |
+| web3-bot | 10 | +99/-17 | 10 |
+| Steven Allen | 1 | +55/-37 | 1 |
+| Jiaying Wang | 5 | +30/-8 | 5 |
+| Jakub Sztandera | 2 | +8/-3 | 3 |
+| Łukasz Magiera | 1 | +3/-3 | 2 |
+| Travis Person | 1 | +2/-2 | 2 |
+| Rod Vagg | 1 | +2/-2 | 2 |
+
 # v1.13.2 /  2022-01-09
 
 Lotus v1.13.2 is a *highly recommended* feature release with remarkable retrieval improvements, new features like 

Makefile

@@ -345,6 +345,8 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen
 
+jen: gen
+
 snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap

api/api_gateway.go

@@ -45,8 +45,9 @@ type Gateway interface {
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)

api/api_net.go

@@ -51,6 +51,10 @@ type Net interface {
 	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
 	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
 
+	NetProtectAdd(ctx context.Context, acl []peer.ID) error    //perm:admin
+	NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
+	NetProtectList(ctx context.Context) ([]peer.ID, error)     //perm:read
+
 	// ResourceManager API
 	NetStat(ctx context.Context, scope string) (NetStat, error)          //perm:read
 	NetLimit(ctx context.Context, scope string) (NetLimit, error)        //perm:read

api/api_storage.go

@@ -222,6 +222,16 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+	IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
@@ -256,7 +266,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }

api/mocks/mock_full.go

@@ -1856,6 +1856,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()

api/proxy_gen.go

@@ -516,6 +516,8 @@ type GatewayStruct struct {
 
 		MsigGetVested func(p0 context.Context, p1 address.Address, p2 types.TipSetKey, p3 types.TipSetKey) (types.BigInt, error) ``
 
+		MsigGetVestingSchedule func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) ``
+
 		StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``
 
 		StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
@@ -593,6 +595,12 @@ type NetStruct struct {
 
 		NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`
 
+		NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
+		NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
+
+		NetProtectRemove func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
 		NetPubsubScores func(p0 context.Context) ([]PubsubScore, error) `perm:"read"`
 
 		NetSetLimit func(p0 context.Context, p1 string, p2 NetLimit) error `perm:"admin"`
@@ -625,7 +633,7 @@ type StorageMinerStruct struct {
 
 		ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"`
 
-		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
+		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
 
 		ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`
 
@@ -639,6 +647,8 @@ type StorageMinerStruct struct {
 
 		DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`
 
+		DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
+
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
 
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -673,6 +683,10 @@ type StorageMinerStruct struct {
 
 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`
 
+		IndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"`
+
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
+
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
 
 		MarketDataTransferDiagnostics func(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) `perm:"write"`
@@ -3285,6 +3299,17 @@ func (s *GatewayStub) MsigGetVested(p0 context.Context, p1 address.Address, p2 t
 	return *new(types.BigInt), ErrNotSupported
 }
 
+func (s *GatewayStruct) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	if s.Internal.MsigGetVestingSchedule == nil {
+		return *new(MsigVesting), ErrNotSupported
+	}
+	return s.Internal.MsigGetVestingSchedule(p0, p1, p2)
+}
+
+func (s *GatewayStub) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	return *new(MsigVesting), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported
@@ -3670,6 +3695,39 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
 	return *new([]peer.AddrInfo), ErrNotSupported
 }
 
+func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectAdd == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectAdd(p0, p1)
+}
+
+func (s *NetStub) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	if s.Internal.NetProtectList == nil {
+		return *new([]peer.ID), ErrNotSupported
+	}
+	return s.Internal.NetProtectList(p0)
+}
+
+func (s *NetStub) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	return *new([]peer.ID), ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectRemove == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectRemove(p0, p1)
+}
+
+func (s *NetStub) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
 func (s *NetStruct) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	if s.Internal.NetPubsubScores == nil {
 		return *new([]PubsubScore), ErrNotSupported
@@ -3747,14 +3805,14 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres
 	return *new(abi.SectorSize), ErrNotSupported
 }
 
-func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	if s.Internal.CheckProvable == nil {
 		return *new(map[abi.SectorNumber]string), ErrNotSupported
 	}
-	return s.Internal.CheckProvable(p0, p1, p2, p3)
+	return s.Internal.CheckProvable(p0, p1, p2, p3, p4)
 }
 
-func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	return *new(map[abi.SectorNumber]string), ErrNotSupported
 }
 
@@ -3824,6 +3882,17 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
 	return *new([]DagstoreShardInfo), ErrNotSupported
 }
 
+func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	if s.Internal.DagstoreLookupPieces == nil {
+		return *new([]DagstoreShardInfo), ErrNotSupported
+	}
+	return s.Internal.DagstoreLookupPieces(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	return *new([]DagstoreShardInfo), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
 	if s.Internal.DagstoreRecoverShard == nil {
 		return ErrNotSupported
@@ -4011,6 +4080,28 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) IndexerAnnounceAllDeals(p0 context.Context) error {
+	if s.Internal.IndexerAnnounceAllDeals == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceAllDeals(p0)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceAllDeals(p0 context.Context) error {
+	return ErrNotSupported
+}
+
+func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.IndexerAnnounceDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
 	if s.Internal.MarketCancelDataTransfer == nil {
 		return ErrNotSupported

api/v0api/v0mocks/mock_full.go

@@ -1769,6 +1769,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }
 
+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()

api/version.go

@@ -57,7 +57,7 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)
 
-	MinerAPIVersion0  = newVer(1, 3, 0)
+	MinerAPIVersion0  = newVer(1, 4, 0)
 	WorkerAPIVersion0 = newVer(1, 5, 0)
 )
 

blockstore/context.go

@@ -0,0 +1,21 @@
+package blockstore
+
+import (
+	"context"
+)
+
+type hotViewKey struct{}
+
+var hotView = hotViewKey{}
+
+// WithHotView constructs a new context with an option that provides a hint to the blockstore
+// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
+func WithHotView(ctx context.Context) context.Context {
+	return context.WithValue(ctx, hotView, struct{}{})
+}
+
+// IsHotView returns true if the hot view option is set in the context
+func IsHotView(ctx context.Context) bool {
+	v := ctx.Value(hotView)
+	return v != nil
+}

blockstore/splitstore/splitstore.go

@@ -161,6 +161,13 @@ type SplitStore struct {
 	txnSyncCond     sync.Cond
 	txnSync         bool
 
+	// background cold object reification
+	reifyWorkers    sync.WaitGroup
+	reifyMx         sync.Mutex
+	reifyCond       sync.Cond
+	reifyPend       map[cid.Cid]struct{}
+	reifyInProgress map[cid.Cid]struct{}
+
 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
 }
@@ -202,6 +209,10 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())
 
+	ss.reifyCond.L = &ss.reifyMx
+	ss.reifyPend = make(map[cid.Cid]struct{})
+	ss.reifyInProgress = make(map[cid.Cid]struct{})
+
 	if enableDebugLog {
 		ss.debug, err = openDebugLog(path)
 		if err != nil {
@@ -264,7 +275,13 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		return true, nil
 	}
 
-	return s.cold.Has(ctx, cid)
+	has, err = s.cold.Has(ctx, cid)
+	if has && bstore.IsHotView(ctx) {
+		s.reifyColdObject(cid)
+	}
+
+	return has, err
+
 }
 
 func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
@@ -308,8 +325,11 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 
 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
 
+			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return blk, err
 
@@ -359,6 +379,10 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 
 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return size, err
@@ -536,6 +560,10 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 
 		err = s.cold.View(ctx, cid, cb)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
@@ -645,6 +673,9 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		}
 	}
 
+	// spawn the reifier
+	go s.reifyOrchestrator()
+
 	// watch the chain
 	chain.SubscribeHeadChanges(s.HeadChange)
 
@@ -676,6 +707,8 @@ func (s *SplitStore) Close() error {
 		}
 	}
 
+	s.reifyCond.Broadcast()
+	s.reifyWorkers.Wait()
 	s.cancel()
 	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
 }

blockstore/splitstore/splitstore_reify.go

@@ -0,0 +1,214 @@
+package splitstore
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+)
+
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
+
+func (s *SplitStore) reifyColdObject(c cid.Cid) {
+	if !s.isWarm() {
+		return
+	}
+
+	if isUnitaryObject(c) {
+		return
+	}
+
+	s.reifyMx.Lock()
+	defer s.reifyMx.Unlock()
+
+	_, ok := s.reifyInProgress[c]
+	if ok {
+		return
+	}
+
+	s.reifyPend[c] = struct{}{}
+	s.reifyCond.Broadcast()
+}
+
+func (s *SplitStore) reifyOrchestrator() {
+	workers := runtime.NumCPU() / 4
+	if workers < 2 {
+		workers = 2
+	}
+
+	workch := make(chan cid.Cid, workers)
+	defer close(workch)
+
+	for i := 0; i < workers; i++ {
+		s.reifyWorkers.Add(1)
+		go s.reifyWorker(workch)
+	}
+
+	for {
+		s.reifyMx.Lock()
+		for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
+			s.reifyCond.Wait()
+		}
+
+		if atomic.LoadInt32(&s.closing) != 0 {
+			s.reifyMx.Unlock()
+			return
+		}
+
+		reifyPend := s.reifyPend
+		s.reifyPend = make(map[cid.Cid]struct{})
+		s.reifyMx.Unlock()
+
+		for c := range reifyPend {
+			select {
+			case workch <- c:
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
+	defer s.reifyWorkers.Done()
+	for c := range workch {
+		s.doReify(c)
+	}
+}
+
+func (s *SplitStore) doReify(c cid.Cid) {
+	var toreify, totrack, toforget []cid.Cid
+
+	defer func() {
+		s.reifyMx.Lock()
+		defer s.reifyMx.Unlock()
+
+		for _, c := range toreify {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range toforget {
+			delete(s.reifyInProgress, c)
+		}
+	}()
+
+	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
+
+	count := 0
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
+			s.reifyMx.Lock()
+			_, inProgress := s.reifyInProgress[c]
+			if !inProgress {
+				s.reifyInProgress[c] = struct{}{}
+			}
+			s.reifyMx.Unlock()
+
+			if inProgress {
+				return errStopWalk
+			}
+
+			has, err := s.hot.Has(s.ctx, c)
+			if err != nil {
+				return xerrors.Errorf("error checking hotstore: %w", err)
+			}
+
+			if has {
+				if s.txnMarkSet != nil {
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
+			}
+
+			toreify = append(toreify, c)
+			return nil
+		},
+		func(missing cid.Cid) error {
+			log.Warnf("missing reference while reifying %s: %s", c, missing)
+			return errStopWalk
+		})
+
+	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
+		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
+		return
+	}
+
+	log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
+
+	// this should not get too big, maybe some 100s of objects.
+	batch := make([]blocks.Block, 0, len(toreify))
+	for _, c := range toreify {
+		blk, err := s.cold.Get(s.ctx, c)
+		if err != nil {
+			log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
+			continue
+		}
+
+		if err := s.checkClosing(); err != nil {
+			return
+		}
+
+		batch = append(batch, blk)
+	}
+
+	if len(batch) > 0 {
+		err = s.hot.PutMany(s.ctx, batch)
+		if err != nil {
+			log.Warnf("error reifying cold object (cid: %s): %s", c, err)
+			return
+		}
+	}
+
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
+}

blockstore/splitstore/splitstore_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"sync"
 	"sync/atomic"
@@ -387,6 +388,235 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 }
 
+func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// first access using the standard view
+	err = f(context.Background(), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nothing should be reified
+	waitForReification()
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+	// now make the hot/reifying view and ensure access reifies
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// everything should be reified
+	waitForReification()
+	for i, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatalf("block%d was not reified", i+1)
+		}
+	}
+}
+
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
+func TestSplitStoreReification(t *testing.T) {
+	t.Log("test reification with Has")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+	t.Log("test reification with Get")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Get(ctx, c)
+		return err
+	})
+	t.Log("test reification with GetSize")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.GetSize(ctx, c)
+		return err
+	})
+	t.Log("test reification with View")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		return s.View(ctx, c, func(_ []byte) error { return nil })
+	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+}
+
 type mockChain struct {
 	t testing.TB
 

blockstore/splitstore/visitor.go

@@ -26,6 +26,10 @@ type tmpVisitor struct {
 var _ ObjectVisitor = (*tmpVisitor)(nil)
 
 func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	return v.set.Visit(c), nil
 }
 
@@ -45,6 +49,10 @@ func newConcurrentVisitor() *concurrentVisitor {
 }
 
 func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	v.mx.Lock()
 	defer v.mx.Unlock()
 

build/params_mainnet.go

@@ -67,7 +67,8 @@ const UpgradeHyperdriveHeight = 892800
 // 2021-10-26T13:30:00Z
 const UpgradeChocolateHeight = 1231620
 
-var UpgradeOhSnapHeight = abi.ChainEpoch(999999999999)
+// 2022-03-01T15:00:00Z
+var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)
 
 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {

build/params_shared_funcs.go

@@ -13,6 +13,17 @@ import (
 
 func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
 func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func IndexerIngestTopic(netName dtypes.NetworkName) string {
+
+	nn := string(netName)
+	// The network name testnetnet is here for historical reasons.
+	// Going forward we aim to use the name `mainnet` where possible.
+	if nn == "testnetnet" {
+		nn = "mainnet"
+	}
+
+	return "/indexer/ingest/" + nn
+}
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.15.0-dev"
+const BuildVersion = "1.15.1-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/builtin/verifreg/actor.go.template

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )
 
 func init() {
@@ -62,6 +63,11 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }
 
+type RemoveDataCapProposal = verifreg{{.latestVersion}}.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg{{.latestVersion}}.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg{{.latestVersion}}.RemoveDataCapParams
+type RmDcProposalID = verifreg{{.latestVersion}}.RmDcProposalID
+const SignatureDomainSeparation_RemoveDataCap = verifreg{{.latestVersion}}.SignatureDomainSeparation_RemoveDataCap
 
 type State interface {
 	cbor.Marshaler
@@ -69,6 +75,7 @@ type State interface {
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}

chain/actors/builtin/verifreg/state.go.template

@@ -61,6 +61,10 @@ func (s *state{{.v}}) VerifierDataCap(addr address.Address) (bool, abi.StoragePo
 	return getDataCap(s.store, actors.Version{{.v}}, s.verifiers, addr)
 }
 
+func (s *state{{.v}}) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version{{.v}}, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state{{.v}}) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version{{.v}}, s.verifiers, cb)
 }
@@ -77,6 +81,11 @@ func (s *state{{.v}}) verifiers() (adt.Map, error) {
 	return adt{{.v}}.AsMap(s.store, s.Verifiers{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
 }
 
+func (s *state{{.v}}) removeDataCapProposalIDs() (adt.Map, error) {
+    {{if le .v 6}}return nil, nil
+    {{else}}return adt{{.v}}.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin{{.v}}.DefaultHamtBitwidth){{end}}
+}
+
 func (s *state{{.v}}) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/util.go

@@ -6,6 +6,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 	"golang.org/x/xerrors"
 )
 
@@ -50,3 +51,28 @@ func forEachCap(store adt.Store, ver actors.Version, root rootFunc, cb func(addr
 		return cb(a, dcap)
 	})
 }
+
+func getRemoveDataCapProposalID(store adt.Store, ver actors.Version, root rootFunc, verifier address.Address, client address.Address) (bool, uint64, error) {
+	if verifier.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	if client.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	vh, err := root()
+	if err != nil {
+		return false, 0, xerrors.Errorf("loading verifreg: %w", err)
+	}
+	if vh == nil {
+		return false, 0, xerrors.Errorf("remove data cap proposal hamt not found. you are probably using an incompatible version of actors")
+	}
+
+	var id verifreg.RmDcProposalID
+	if found, err := vh.Get(abi.NewAddrPairKey(verifier, client), &id); err != nil {
+		return false, 0, xerrors.Errorf("looking up addr pair: %w", err)
+	} else if !found {
+		return false, 0, nil
+	}
+
+	return true, id.ProposalID, nil
+}

chain/actors/builtin/verifreg/v0.go

@@ -53,6 +53,10 @@ func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version0, s.verifiers, addr)
 }
 
+func (s *state0) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version0, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version0, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state0) verifiers() (adt.Map, error) {
 	return adt0.AsMap(s.store, s.Verifiers)
 }
 
+func (s *state0) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state0) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v2.go

@@ -53,6 +53,10 @@ func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version2, s.verifiers, addr)
 }
 
+func (s *state2) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version2, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version2, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state2) verifiers() (adt.Map, error) {
 	return adt2.AsMap(s.store, s.Verifiers)
 }
 
+func (s *state2) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state2) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v3.go

@@ -54,6 +54,10 @@ func (s *state3) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version3, s.verifiers, addr)
 }
 
+func (s *state3) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version3, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state3) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version3, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state3) verifiers() (adt.Map, error) {
 	return adt3.AsMap(s.store, s.Verifiers, builtin3.DefaultHamtBitwidth)
 }
 
+func (s *state3) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state3) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v4.go

@@ -54,6 +54,10 @@ func (s *state4) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version4, s.verifiers, addr)
 }
 
+func (s *state4) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version4, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state4) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version4, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state4) verifiers() (adt.Map, error) {
 	return adt4.AsMap(s.store, s.Verifiers, builtin4.DefaultHamtBitwidth)
 }
 
+func (s *state4) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state4) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v5.go

@@ -54,6 +54,10 @@ func (s *state5) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version5, s.verifiers, addr)
 }
 
+func (s *state5) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version5, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state5) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version5, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state5) verifiers() (adt.Map, error) {
 	return adt5.AsMap(s.store, s.Verifiers, builtin5.DefaultHamtBitwidth)
 }
 
+func (s *state5) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state5) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v6.go

@@ -54,6 +54,10 @@ func (s *state6) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version6, s.verifiers, addr)
 }
 
+func (s *state6) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version6, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state6) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version6, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state6) verifiers() (adt.Map, error) {
 	return adt6.AsMap(s.store, s.Verifiers, builtin6.DefaultHamtBitwidth)
 }
 
+func (s *state6) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state6) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/v7.go

@@ -54,6 +54,10 @@ func (s *state7) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version7, s.verifiers, addr)
 }
 
+func (s *state7) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version7, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state7) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version7, s.verifiers, cb)
 }
@@ -70,6 +74,10 @@ func (s *state7) verifiers() (adt.Map, error) {
 	return adt7.AsMap(s.store, s.Verifiers, builtin7.DefaultHamtBitwidth)
 }
 
+func (s *state7) removeDataCapProposalIDs() (adt.Map, error) {
+	return adt7.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin7.DefaultHamtBitwidth)
+}
+
 func (s *state7) GetState() interface{} {
 	return &s.State
 }

chain/actors/builtin/verifreg/verifreg.go

@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )
 
 func init() {
@@ -151,12 +152,20 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }
 
+type RemoveDataCapProposal = verifreg7.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg7.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg7.RemoveDataCapParams
+type RmDcProposalID = verifreg7.RmDcProposalID
+
+const SignatureDomainSeparation_RemoveDataCap = verifreg7.SignatureDomainSeparation_RemoveDataCap
+
 type State interface {
 	cbor.Marshaler
 
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}

chain/consensus/filcns/compute_state.go

@@ -32,6 +32,7 @@ import (
 
 	/* inline-gen end */
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -92,6 +93,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 		partDone()
 	}()
 
+	ctx = blockstore.WithHotView(ctx)
 	makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (*vm.VM, error) {
 		vmopt := &vm.VMOpts{
 			StateBase:      base,

chain/consensus/filcns/filecoin.go

@@ -467,7 +467,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 	}
 
 	nv := filec.sm.GetNetworkVersion(ctx, b.Header.Height)
-	pl := vm.PricelistByEpoch(baseTs.Height())
+	pl := vm.PricelistByEpoch(b.Header.Height)
 	var sumGasLimit int64
 	checkMsg := func(msg types.ChainMsg) error {
 		m := msg.VMMessage()

chain/consensus/filcns/upgrades.go

@@ -165,13 +165,8 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
 		Migration: UpgradeActorsV7,
 		PreMigrations: []stmgr.PreMigration{{
 			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     120,
+			StartWithin:     180,
 			DontStartWithin: 60,
-			StopWithin:      35,
-		}, {
-			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     30,
-			DontStartWithin: 15,
 			StopWithin:      5,
 		}},
 		Expensive: true,
@@ -1264,7 +1259,7 @@ func upgradeActorsV7Common(
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
 	config nv15.Config,
 ) (cid.Cid, error) {
-	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB)
+	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4)
 	// TODO: pretty sure we'd achieve nothing by doing this, confirm in review
 	//buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), writeStore)
 	store := store.ActorStore(ctx, writeStore)

chain/messagepool/check.go

@@ -106,7 +106,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 	curTs := mp.curTs
 	mp.curTsLk.Unlock()
 
-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1
 
 	var baseFee big.Int
 	if len(curTs.Blocks()) > 0 {

chain/messagepool/check_test.go

@@ -0,0 +1,224 @@
+//stm: #unit
+package messagepool
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
+	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
+)
+
+func init() {
+	_ = logging.SetLogLevel("*", "INFO")
+}
+
+func getCheckMessageStatus(statusCode api.CheckStatusCode, msgStatuses []api.MessageCheckStatus) (*api.MessageCheckStatus, error) {
+	for i := 0; i < len(msgStatuses); i++ {
+		iMsgStatuses := msgStatuses[i]
+		if iMsgStatuses.CheckStatus.Code == statusCode {
+			return &iMsgStatuses, nil
+		}
+	}
+	return nil, fmt.Errorf("Could not find CheckStatusCode %s", statusCode)
+}
+
+func TestCheckMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	var protos []*api.MessagePrototype
+	for i := 0; i < 5; i++ {
+		msg := &types.Message{
+			To:         target,
+			From:       sender,
+			Value:      types.NewInt(1),
+			Nonce:      uint64(i),
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 2<<10),
+		}
+		proto := &api.MessagePrototype{
+			Message:    *msg,
+			ValidNonce: true,
+		}
+		protos = append(protos, proto)
+	}
+
+	messageStatuses, err := mp.CheckMessages(context.TODO(), protos)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckPendingMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_PENDING_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	messageStatuses, err := mp.CheckPendingMessages(context.TODO(), sender)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckReplaceMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_REPLACE_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	// create a new message with the same data, except that it is too big
+	var msgs []*types.Message
+	invalidmsg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 128<<10),
+	}
+	msgs = append(msgs, invalidmsg)
+
+	{
+		messageStatuses, err := mp.CheckReplaceMessages(context.TODO(), msgs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < len(messageStatuses); i++ {
+			iMsgStatuses := messageStatuses[i]
+
+			status, err := getCheckMessageStatus(api.CheckStatusMessageSize, iMsgStatuses)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// the replacement message should cause a status error
+			assert.False(t, status.OK)
+		}
+	}
+
+}

chain/messagepool/messagepool.go

@@ -628,7 +628,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
 // For non local messages, if the message cannot be included in the next 20 blocks it returns
 // a (soft) validation error.
 func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1
 	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())
 
 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {

chain/messagepool/messagepool_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -226,6 +227,8 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) {
 }
 
 func TestMessagePool(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
+
 	tma := newTestMpoolAPI()
 
 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -327,6 +330,7 @@ func TestCheckMessageBig(t *testing.T) {
 			Message:   *msg,
 			Signature: *sig,
 		}
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		err = mp.Add(context.TODO(), sm)
 		assert.ErrorIs(t, err, ErrMessageTooBig)
 	}
@@ -760,3 +764,302 @@ func TestUpdates(t *testing.T) {
 		t.Fatal("expected closed channel, but got an update instead")
 	}
 }
+
+func TestMessageBelowMinGasFee(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	// fee is just below minimum gas fee
+	fee := minimumBaseFee.Uint64() - 1
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(fee),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.ErrorIs(t, err, ErrGasFeeCapTooLow)
+	}
+}
+
+func TestMessageValueTooHigh(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	totalFil := types.TotalFilecoinInt
+	extra := types.NewInt(1)
+
+	value := types.BigAdd(totalFil, extra)
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      value,
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+	}
+}
+
+func TestMessageSignatureInvalid(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		badSig := &crypto.Signature{
+			Type: crypto.SigTypeSecp256k1,
+			Data: make([]byte, 0),
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *badSig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+		// assert.Contains(t, err.Error(), "invalid signature length")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwice(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create a valid messages
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// try to add it twice
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "with nonce 0 already in mpool")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceNonceGap(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create message with invalid nonce (1)
+		sm := makeTestMessage(w, from, to, 1, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "unfulfilled nonce gap")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiff(t *testing.T) {
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_001, minimumBaseFee.Uint64())
+
+		//stm: @CHAIN_MEMPOOL_PUSH_001
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm2)
+		// assert.Contains(t, err.Error(), "replace by fee has too low GasPremium")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiffReplaced(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64()*2)
+		mustAdd(t, mp, sm2)
+	}
+}
+
+func TestRemoveMessage(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		//stm: @CHAIN_MEMPOOL_REMOVE_001
+		// remove message for sender
+		mp.Remove(context.TODO(), from, sm.Message.Nonce, true)
+
+		//stm: @CHAIN_MEMPOOL_PENDING_FOR_001
+		// check messages in pool: should be none present
+		msgs := mp.pendingFor(context.TODO(), from)
+		assert.Len(t, msgs, 0)
+	}
+}

chain/messagepool/repub_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool
 
 import (
@@ -16,6 +17,7 @@ import (
 )
 
 func TestRepubMessages(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
 	oldRepublishBatchDelay := RepublishBatchDelay
 	RepublishBatchDelay = time.Microsecond
 	defer func() {
@@ -57,6 +59,7 @@ func TestRepubMessages(t *testing.T) {
 
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)

chain/messagepool/selection_test.go

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool
 
 import (
@@ -74,6 +75,8 @@ func makeTestMpool() (*MessagePool, *testMpoolAPI) {
 }
 
 func TestMessageChains(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
+	//stm: @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -310,6 +313,8 @@ func TestMessageChains(t *testing.T) {
 }
 
 func TestMessageChainSkipping(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
+
 	// regression test for chain skip bug
 
 	mp, tma := makeTestMpool()
@@ -382,6 +387,7 @@ func TestMessageChainSkipping(t *testing.T) {
 }
 
 func TestBasicMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	oldMaxNonceGap := MaxNonceGap
 	MaxNonceGap = 1000
 	defer func() {
@@ -532,6 +538,7 @@ func TestBasicMessageSelection(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingGas(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -595,6 +602,7 @@ func TestMessageSelectionTrimmingGas(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -641,6 +649,7 @@ func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -707,6 +716,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
 }
 
 func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -788,6 +798,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
 }
 
 func TestPriorityMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -867,6 +878,7 @@ func TestPriorityMessageSelection(t *testing.T) {
 }
 
 func TestPriorityMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -934,6 +946,7 @@ func TestPriorityMessageSelection2(t *testing.T) {
 }
 
 func TestPriorityMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()
 
 	// the actors
@@ -1028,6 +1041,8 @@ func TestPriorityMessageSelection3(t *testing.T) {
 }
 
 func TestOptimalMessageSelection1(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor
 	// from the start
@@ -1094,6 +1109,8 @@ func TestOptimalMessageSelection1(t *testing.T) {
 }
 
 func TestOptimalMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses two actors sending messages to each other, with the first
 	// actor paying (much) higher gas premium than the second.
 	// We select with a low ticket quality; the chain depenent merging algorithm should pick
@@ -1173,6 +1190,8 @@ func TestOptimalMessageSelection2(t *testing.T) {
 }
 
 func TestOptimalMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses 10 actors sending a block of messages to each other, with the the first
 	// actors paying higher gas premium than the subsequent actors.
 	// We select with a low ticket quality; the chain dependent merging algorithm should pick
@@ -1416,6 +1435,8 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
 }
 
 func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1439,6 +1460,8 @@ func TestCompetitiveMessageSelectionExp(t *testing.T) {
 }
 
 func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1462,6 +1485,7 @@ func TestCompetitiveMessageSelectionZipf(t *testing.T) {
 }
 
 func TestGasReward(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_GAS_REWARD_001
 	tests := []struct {
 		Premium   uint64
 		FeeCap    uint64
@@ -1494,6 +1518,8 @@ func TestGasReward(t *testing.T) {
 }
 
 func TestRealWorldSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @TOKEN_WALLET_SIGN_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// load test-messages.json.gz and rewrite the messages so that
 	// 1) we map each real actor to a test actor so that we can sign the messages
 	// 2) adjust the nonces so that they start from 0

chain/sub/incoming.go

@@ -1,19 +1,24 @@
 package sub
 
 import (
+	"bytes"
 	"context"
-	"fmt"
+	"encoding/binary"
+	"sync"
 	"time"
 
 	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/sub/ratelimit"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/impl/client"
+	"github.com/filecoin-project/lotus/node/impl/full"
 	lru "github.com/hashicorp/golang-lru"
 	blocks "github.com/ipfs/go-block-format"
 	bserv "github.com/ipfs/go-blockservice"
@@ -168,12 +173,12 @@ func fetchCids(
 	cidIndex := make(map[cid.Cid]int)
 	for i, c := range cids {
 		if c.Prefix() != msgCidPrefix {
-			return fmt.Errorf("invalid msg CID: %s", c)
+			return xerrors.Errorf("invalid msg CID: %s", c)
 		}
 		cidIndex[c] = i
 	}
 	if len(cids) != len(cidIndex) {
-		return fmt.Errorf("duplicate CIDs in fetchCids input")
+		return xerrors.Errorf("duplicate CIDs in fetchCids input")
 	}
 
 	for block := range bserv.GetBlocks(ctx, cids) {
@@ -196,7 +201,7 @@ func fetchCids(
 	if len(cidIndex) > 0 {
 		err := ctx.Err()
 		if err == nil {
-			err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
+			err = xerrors.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
 		}
 		return err
 	}
@@ -444,3 +449,168 @@ func recordFailure(ctx context.Context, metric *stats.Int64Measure, failureType
 	)
 	stats.Record(ctx, metric.M(1))
 }
+
+type peerMsgInfo struct {
+	peerID    peer.ID
+	lastCid   cid.Cid
+	lastSeqno uint64
+	rateLimit *ratelimit.Window
+	mutex     sync.Mutex
+}
+
+type IndexerMessageValidator struct {
+	self peer.ID
+
+	peerCache *lru.TwoQueueCache
+	chainApi  full.ChainModuleAPI
+	stateApi  full.StateModuleAPI
+}
+
+func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator {
+	peerCache, _ := lru.New2Q(8192)
+
+	return &IndexerMessageValidator{
+		self:      self,
+		peerCache: peerCache,
+		chainApi:  chainApi,
+		stateApi:  stateApi,
+	}
+}
+
+func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
+	// This chain-node should not be publishing its own messages.  These are
+	// relayed from market-nodes.  If a node appears to be local, reject it.
+	if pid == v.self {
+		log.Debug("ignoring indexer message from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+	originPeer := msg.GetFrom()
+	if originPeer == v.self {
+		log.Debug("ignoring indexer message originating from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+
+	idxrMsg := dtsync.Message{}
+	err := idxrMsg.UnmarshalCBOR(bytes.NewBuffer(msg.Data))
+	if err != nil {
+		log.Errorw("Could not decode indexer pubsub message", "err", err)
+		return pubsub.ValidationReject
+	}
+	if len(idxrMsg.ExtraData) == 0 {
+		log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
+		return pubsub.ValidationIgnore
+	}
+
+	// Get miner info from lotus
+	minerAddr, err := address.NewFromBytes(idxrMsg.ExtraData)
+	if err != nil {
+		log.Warnw("cannot parse extra data as miner address", "err", err, "extraData", idxrMsg.ExtraData)
+		return pubsub.ValidationReject
+	}
+
+	minerID := minerAddr.String()
+	msgCid := idxrMsg.Cid
+
+	var msgInfo *peerMsgInfo
+	val, ok := v.peerCache.Get(minerID)
+	if !ok {
+		msgInfo = &peerMsgInfo{}
+	} else {
+		msgInfo = val.(*peerMsgInfo)
+	}
+
+	// Lock this peer's message info.
+	msgInfo.mutex.Lock()
+	defer msgInfo.mutex.Unlock()
+
+	if ok {
+		// Reject replayed messages.
+		seqno := binary.BigEndian.Uint64(msg.Message.GetSeqno())
+		if seqno <= msgInfo.lastSeqno {
+			log.Debugf("ignoring replayed indexer message")
+			return pubsub.ValidationIgnore
+		}
+		msgInfo.lastSeqno = seqno
+	}
+
+	if !ok || originPeer != msgInfo.peerID {
+		// Check that the miner ID maps to the peer that sent the message.
+		err = v.authenticateMessage(ctx, minerAddr, originPeer)
+		if err != nil {
+			log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID)
+			stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+			return pubsub.ValidationReject
+		}
+		msgInfo.peerID = originPeer
+		if !ok {
+			// Add msgInfo to cache only after being authenticated.  If two
+			// messages from the same peer are handled concurrently, there is a
+			// small chance that one msgInfo could replace the other here when
+			// the info is first cached.  This is OK, so no need to prevent it.
+			v.peerCache.Add(minerID, msgInfo)
+		}
+	}
+
+	// See if message needs to be ignored due to rate limiting.
+	if v.rateLimitPeer(msgInfo, msgCid) {
+		return pubsub.ValidationIgnore
+	}
+
+	stats.Record(ctx, metrics.IndexerMessageValidationSuccess.M(1))
+	return pubsub.ValidationAccept
+}
+
+func (v *IndexerMessageValidator) rateLimitPeer(msgInfo *peerMsgInfo, msgCid cid.Cid) bool {
+	const (
+		msgLimit        = 5
+		msgTimeLimit    = 10 * time.Second
+		repeatTimeLimit = 2 * time.Hour
+	)
+
+	timeWindow := msgInfo.rateLimit
+
+	// Check overall message rate.
+	if timeWindow == nil {
+		timeWindow = ratelimit.NewWindow(msgLimit, msgTimeLimit)
+		msgInfo.rateLimit = timeWindow
+	} else if msgInfo.lastCid == msgCid {
+		// Check if this is a repeat of the previous message data.
+		if time.Since(timeWindow.Newest()) < repeatTimeLimit {
+			log.Warnw("ignoring repeated indexer message", "sender", msgInfo.peerID)
+			return true
+		}
+	}
+
+	err := timeWindow.Add()
+	if err != nil {
+		log.Warnw("ignoring indexer message", "sender", msgInfo.peerID, "err", err)
+		return true
+	}
+
+	msgInfo.lastCid = msgCid
+
+	return false
+}
+
+func (v *IndexerMessageValidator) authenticateMessage(ctx context.Context, minerAddress address.Address, peerID peer.ID) error {
+	ts, err := v.chainApi.ChainHead(ctx)
+	if err != nil {
+		return err
+	}
+
+	minerInfo, err := v.stateApi.StateMinerInfo(ctx, minerAddress, ts.Key())
+	if err != nil {
+		return err
+	}
+
+	if minerInfo.PeerId == nil {
+		return xerrors.New("no peer id for miner")
+	}
+	if *minerInfo.PeerId != peerID {
+		return xerrors.New("miner id does not map to peer that sent message")
+	}
+
+	return nil
+}

chain/sub/incoming_test.go

@@ -2,13 +2,20 @@
 package sub
 
 import (
+	"bytes"
 	"context"
 	"testing"
 
-	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
+	"github.com/filecoin-project/lotus/api/mocks"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/golang/mock/gomock"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )
 
 type getter struct {
@@ -63,3 +70,65 @@ func TestFetchCidsWithDedup(t *testing.T) {
 		t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1])
 	}
 }
+
+func TestIndexerMessageValidator_Validate(t *testing.T) {
+	validCid, err := cid.Decode("QmbpDgg5kRLDgMxS8vPKNFXEcA6D5MC4CkuUdSWDVtHPGK")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name           string
+		selfPID        string
+		senderPID      string
+		extraData      []byte
+		wantValidation pubsub.ValidationResult
+	}{
+		{
+			name:           "invalid extra data is rejected",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ",
+			extraData:      []byte("f0127896"), // note, casting encoded address to byte is invalid.
+			wantValidation: pubsub.ValidationReject,
+		},
+		{
+			name:           "same sender and receiver is ignored",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			wantValidation: pubsub.ValidationIgnore,
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			mc := gomock.NewController(t)
+			node := mocks.NewMockFullNode(mc)
+			subject := NewIndexerMessageValidator(peer.ID(tc.selfPID), node, node)
+			message := dtsync.Message{
+				Cid:       validCid,
+				Addrs:     nil,
+				ExtraData: tc.extraData,
+			}
+			buf := bytes.NewBuffer(nil)
+			if err := message.MarshalCBOR(buf); err != nil {
+				t.Fatal(err)
+			}
+
+			topic := "topic"
+			pbm := &pb.Message{
+				Data:  buf.Bytes(),
+				Topic: &topic,
+				From:  nil,
+				Seqno: nil,
+			}
+			validate := subject.Validate(context.Background(), peer.ID(tc.senderPID), &pubsub.Message{
+				Message:       pbm,
+				ReceivedFrom:  peer.ID(tc.senderPID),
+				ValidatorData: nil,
+			})
+
+			if validate != tc.wantValidation {
+				t.Fatalf("expected %v but got %v", tc.wantValidation, validate)
+			}
+		})
+	}
+}

chain/sub/ratelimit/queue.go

@@ -0,0 +1,89 @@
+package ratelimit
+
+import "errors"
+
+var ErrRateLimitExceeded = errors.New("rate limit exceeded")
+
+type queue struct {
+	buf   []int64
+	count int
+	head  int
+	tail  int
+}
+
+// cap returns the queue capacity
+func (q *queue) cap() int {
+	return len(q.buf)
+}
+
+// len returns the number of items in the queue
+func (q *queue) len() int {
+	return q.count
+}
+
+// push adds an element to the end of the queue.
+func (q *queue) push(elem int64) error {
+	if q.count == len(q.buf) {
+		return ErrRateLimitExceeded
+	}
+
+	q.buf[q.tail] = elem
+	// Calculate new tail position.
+	q.tail = q.next(q.tail)
+	q.count++
+	return nil
+}
+
+// pop removes and returns the element from the front of the queue.
+func (q *queue) pop() int64 {
+	if q.count <= 0 {
+		panic("pop from empty queue")
+	}
+	ret := q.buf[q.head]
+
+	// Calculate new head position.
+	q.head = q.next(q.head)
+	q.count--
+
+	return ret
+}
+
+// front returns the element at the front of the queue.  This is the element
+// that would be returned by pop().  This call panics if the queue is empty.
+func (q *queue) front() int64 {
+	if q.count <= 0 {
+		panic("front() called when empty")
+	}
+	return q.buf[q.head]
+}
+
+// back returns the element at the back of the queue.  This call panics if the
+// queue is empty.
+func (q *queue) back() int64 {
+	if q.count <= 0 {
+		panic("back() called when empty")
+	}
+	return q.buf[q.prev(q.tail)]
+}
+
+// prev returns the previous buffer position wrapping around buffer.
+func (q *queue) prev(i int) int {
+	if i == 0 {
+		return len(q.buf) - 1
+	}
+	return (i - 1) % len(q.buf)
+}
+
+// next returns the next buffer position wrapping around buffer.
+func (q *queue) next(i int) int {
+	return (i + 1) % len(q.buf)
+}
+
+// truncate pops values that are less than or equal the specified threshold.
+func (q *queue) truncate(threshold int64) {
+	for q.count != 0 && q.buf[q.head] <= threshold {
+		// pop() without returning a value
+		q.head = q.next(q.head)
+		q.count--
+	}
+}

chain/sub/ratelimit/window.go

@@ -0,0 +1,70 @@
+package ratelimit
+
+import "time"
+
+// Window is a time windows for counting events within a span of time.  The
+// windows slides forward in time so that it spans from the most recent event
+// to size time in the past.
+type Window struct {
+	q    *queue
+	size int64
+}
+
+// NewWindow creates a new Window that limits the number of events to maximum
+// count of events within a duration of time.  The capacity sets the maximum
+// number of events, and size sets the span of time over which the events are
+// counted.
+func NewWindow(capacity int, size time.Duration) *Window {
+	return &Window{
+		q: &queue{
+			buf: make([]int64, capacity),
+		},
+		size: int64(size),
+	}
+}
+
+// Add attempts to append a new timestamp into the current window.  Previously
+// added values that are not not within `size` difference from the value being
+// added are first removed.  Add fails if adding the value would cause the
+// window to exceed capacity.
+func (w *Window) Add() error {
+	now := time.Now().UnixNano()
+	if w.Len() != 0 {
+		w.q.truncate(now - w.size)
+	}
+	return w.q.push(now)
+}
+
+// Cap returns the maximum number of items the window can hold.
+func (w *Window) Cap() int {
+	return w.q.cap()
+}
+
+// Len returns the number of elements currently in the window.
+func (w *Window) Len() int {
+	return w.q.len()
+}
+
+// Span returns the distance from the first to the last item in the window.
+func (w *Window) Span() time.Duration {
+	if w.q.len() < 2 {
+		return 0
+	}
+	return time.Duration(w.q.back() - w.q.front())
+}
+
+// Oldest returns the oldest timestamp in the window.
+func (w *Window) Oldest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.front())
+}
+
+// Newest returns the newest timestamp in the window.
+func (w *Window) Newest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.back())
+}

chain/sub/ratelimit/window_test.go

@@ -0,0 +1,61 @@
+package ratelimit
+
+import (
+	"testing"
+	"time"
+)
+
+func TestWindow(t *testing.T) {
+	const (
+		maxEvents = 3
+		timeLimit = 100 * time.Millisecond
+	)
+	w := NewWindow(maxEvents, timeLimit)
+	if w.Len() != 0 {
+		t.Fatal("q.Len() =", w.Len(), "expect 0")
+	}
+	if w.Cap() != maxEvents {
+		t.Fatal("q.Cap() =", w.Cap(), "expect 3")
+	}
+	if !w.Newest().IsZero() {
+		t.Fatal("expected newest to be zero time with empty window")
+	}
+	if !w.Oldest().IsZero() {
+		t.Fatal("expected oldest to be zero time with empty window")
+	}
+	if w.Span() != 0 {
+		t.Fatal("expected span to be zero time with empty window")
+	}
+
+	var err error
+	for i := 0; i < maxEvents; i++ {
+		err = w.Add()
+		if err != nil {
+			t.Fatalf("cannot add event %d", i)
+		}
+	}
+	if w.Len() != maxEvents {
+		t.Fatalf("q.Len() is %d, expected %d", w.Len(), maxEvents)
+	}
+	if err = w.Add(); err != ErrRateLimitExceeded {
+		t.Fatalf("add event %d within time limit should have failed with err: %s", maxEvents+1, ErrRateLimitExceeded)
+	}
+
+	time.Sleep(timeLimit)
+	if err = w.Add(); err != nil {
+		t.Fatalf("cannot add event after time limit: %s", err)
+	}
+
+	prev := w.Newest()
+	time.Sleep(timeLimit)
+	err = w.Add()
+	if err != nil {
+		t.Fatalf("cannot add event")
+	}
+	if w.Newest().Before(prev) {
+		t.Fatal("newest is before previous value")
+	}
+	if w.Oldest().Before(prev) {
+		t.Fatal("oldest is before previous value")
+	}
+}

chain/types/mock/chain.go

@@ -3,6 +3,7 @@ package mock
 import (
 	"context"
 	"fmt"
+	"math/rand"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -24,15 +25,7 @@ func Address(i uint64) address.Address {
 }
 
 func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage {
-	msg := &types.Message{
-		To:         to,
-		From:       from,
-		Value:      types.NewInt(1),
-		Nonce:      nonce,
-		GasLimit:   1000000,
-		GasFeeCap:  types.NewInt(100),
-		GasPremium: types.NewInt(1),
-	}
+	msg := UnsignedMessage(from, to, nonce)
 
 	sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
 	if err != nil {
@@ -96,3 +89,35 @@ func TipSet(blks ...*types.BlockHeader) *types.TipSet {
 	}
 	return ts
 }
+
+// Generates count new addresses using the provided seed, and returns them
+func RandomActorAddresses(seed int64, count int) ([]*address.Address, error) {
+	randAddrs := make([]*address.Address, count)
+	source := rand.New(rand.NewSource(seed))
+	for i := 0; i < count; i++ {
+		bytes := make([]byte, 32)
+		_, err := source.Read(bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		addr, err := address.NewActorAddress(bytes)
+		if err != nil {
+			return nil, err
+		}
+		randAddrs[i] = &addr
+	}
+	return randAddrs, nil
+}
+
+func UnsignedMessage(from, to address.Address, nonce uint64) *types.Message {
+	return &types.Message{
+		To:         to,
+		From:       from,
+		Value:      types.NewInt(1),
+		Nonce:      nonce,
+		GasLimit:   1000000,
+		GasFeeCap:  types.NewInt(100),
+		GasPremium: types.NewInt(1),
+	}
+}

chain/wallet/multi_test.go

@@ -0,0 +1,73 @@
+//stm: #unit
+package wallet
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+func TestMultiWallet(t *testing.T) {
+
+	ctx := context.Background()
+
+	local, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var wallet api.Wallet = MultiWallet{
+		Local: local,
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_NEW_ADDRESS_001
+	a1, err := wallet.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_HAS_001
+	exists, err := wallet.WalletHas(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !exists {
+		t.Fatalf("address doesn't exist in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_LIST_001
+	addrs, err := wallet.WalletList(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// one default address and one newly created
+	if len(addrs) == 2 {
+		t.Fatalf("wrong number of addresses in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_EXPORT_001
+	keyInfo, err := wallet.WalletExport(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_IMPORT_001
+	addr, err := wallet.WalletImport(ctx, keyInfo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if addr != a1 {
+		t.Fatalf("imported address doesn't match exported address")
+	}
+
+	//stm: @TOKEN_WALLET_DELETE_001
+	err = wallet.WalletDelete(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

chain/wallet/wallet_test.go

@@ -0,0 +1,105 @@
+//stm: #unit
+package wallet
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWallet(t *testing.T) {
+
+	ctx := context.Background()
+
+	w1, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_NEW_001
+	a1, err := w1.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_HAS_001
+	exists, err := w1.WalletHas(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !exists {
+		t.Fatalf("address doesn't exist in wallet")
+	}
+
+	w2, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a3, err := w2.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_LIST_001
+	addrs, err := w2.WalletList(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(addrs) != 2 {
+		t.Fatalf("wrong number of addresses in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_DELETE_001
+	err = w2.WalletDelete(ctx, a2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_HAS_001
+	exists, err = w2.WalletHas(ctx, a2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if exists {
+		t.Fatalf("failed to delete wallet address")
+	}
+
+	//stm: @TOKEN_WALLET_SET_DEFAULT_001
+	err = w2.SetDefault(a3)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_DEFAULT_ADDRESS_001
+	def, err := w2.GetDefault()
+	if !assert.Equal(t, a3, def) {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_EXPORT_001
+	keyInfo, err := w2.WalletExport(ctx, a3)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_IMPORT_001
+	addr, err := w2.WalletImport(ctx, keyInfo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if addr != a3 {
+		t.Fatalf("imported address doesn't match exported address")
+	}
+
+}

cli/chain.go

@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
@@ -67,6 +68,8 @@ var ChainHeadCmd = &cli.Command{
 	Name:  "head",
 	Usage: "Print chain head",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -80,7 +83,7 @@ var ChainHeadCmd = &cli.Command{
 		}
 
 		for _, c := range head.Cids() {
-			fmt.Println(c)
+			afmt.Println(c)
 		}
 		return nil
 	},
@@ -97,6 +100,8 @@ var ChainGetBlock = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -124,7 +129,7 @@ var ChainGetBlock = &cli.Command{
 				return err
 			}
 
-			fmt.Println(string(out))
+			afmt.Println(string(out))
 			return nil
 		}
 
@@ -163,9 +168,8 @@ var ChainGetBlock = &cli.Command{
 			return err
 		}
 
-		fmt.Println(string(out))
+		afmt.Println(string(out))
 		return nil
-
 	},
 }
 
@@ -182,6 +186,8 @@ var ChainReadObjCmd = &cli.Command{
 	Usage:     "Read the raw bytes of an object",
 	ArgsUsage: "[objectCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -199,7 +205,7 @@ var ChainReadObjCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Printf("%x\n", obj)
+		afmt.Printf("%x\n", obj)
 		return nil
 	},
 }
@@ -215,6 +221,8 @@ var ChainDeleteObjCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -236,7 +244,7 @@ var ChainDeleteObjCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Printf("Obj %s deleted\n", c.String())
+		afmt.Printf("Obj %s deleted\n", c.String())
 		return nil
 	},
 }
@@ -257,6 +265,7 @@ var ChainStatObjCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -282,8 +291,8 @@ var ChainStatObjCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Printf("Links: %d\n", stats.Links)
-		fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
+		afmt.Printf("Links: %d\n", stats.Links)
+		afmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
 		return nil
 	},
 }
@@ -293,6 +302,8 @@ var ChainGetMsgCmd = &cli.Command{
 	Usage:     "Get and print a message by its cid",
 	ArgsUsage: "[messageCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must pass a cid of a message to get")
 		}
@@ -331,7 +342,7 @@ var ChainGetMsgCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Println(string(enc))
+		afmt.Println(string(enc))
 		return nil
 	},
 }
@@ -406,6 +417,7 @@ var ChainInspectUsage = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -507,23 +519,23 @@ var ChainInspectUsage = &cli.Command{
 
 		numRes := cctx.Int("num-results")
 
-		fmt.Printf("Total Gas Limit: %d\n", sum)
-		fmt.Printf("By Sender:\n")
+		afmt.Printf("Total Gas Limit: %d\n", sum)
+		afmt.Printf("By Sender:\n")
 		for i := 0; i < numRes && i < len(senderVals); i++ {
 			sv := senderVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
 		}
-		fmt.Println()
-		fmt.Printf("By Receiver:\n")
+		afmt.Println()
+		afmt.Printf("By Receiver:\n")
 		for i := 0; i < numRes && i < len(destVals); i++ {
 			sv := destVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
 		}
-		fmt.Println()
-		fmt.Printf("By Method:\n")
+		afmt.Println()
+		afmt.Printf("By Method:\n")
 		for i := 0; i < numRes && i < len(methodVals); i++ {
 			sv := methodVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
 		}
 
 		return nil
@@ -548,6 +560,7 @@ var ChainListCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -595,7 +608,7 @@ var ChainListCmd = &cli.Command{
 			tss = otss
 			for i, ts := range tss {
 				pbf := ts.Blocks()[0].ParentBaseFee
-				fmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))
+				afmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))
 
 				for _, b := range ts.Blocks() {
 					msgs, err := api.ChainGetBlockMessages(ctx, b.Cid())
@@ -621,7 +634,7 @@ var ChainListCmd = &cli.Command{
 						avgpremium = big.Div(psum, big.NewInt(int64(lenmsgs)))
 					}
 
-					fmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
+					afmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
 				}
 				if i < len(tss)-1 {
 					msgs, err := api.ChainGetParentMessages(ctx, tss[i+1].Blocks()[0].Cid())
@@ -646,13 +659,13 @@ var ChainListCmd = &cli.Command{
 					gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
 					gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)
 
-					fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
+					afmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
 				}
-				fmt.Println()
+				afmt.Println()
 			}
 		} else {
 			for i := len(tss) - 1; i >= 0; i-- {
-				printTipSet(cctx.String("format"), tss[i])
+				printTipSet(cctx.String("format"), tss[i], afmt)
 			}
 		}
 		return nil
@@ -707,6 +720,8 @@ var ChainGetCmd = &cli.Command{
    - account-state
 `,
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -725,7 +740,7 @@ var ChainGetCmd = &cli.Command{
 
 			p = "/ipfs/" + ts.ParentState().String() + p
 			if cctx.Bool("verbose") {
-				fmt.Println(p)
+				afmt.Println(p)
 			}
 		}
 
@@ -740,7 +755,7 @@ var ChainGetCmd = &cli.Command{
 			if err != nil {
 				return err
 			}
-			fmt.Println(string(b))
+			afmt.Println(string(b))
 			return nil
 		}
 
@@ -782,7 +797,7 @@ var ChainGetCmd = &cli.Command{
 		}
 
 		if cbu == nil {
-			fmt.Printf("%x", raw)
+			afmt.Printf("%x", raw)
 			return nil
 		}
 
@@ -794,7 +809,7 @@ var ChainGetCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		fmt.Println(string(b))
+		afmt.Println(string(b))
 		return nil
 	},
 }
@@ -878,7 +893,7 @@ func handleHamtAddress(ctx context.Context, api v0api.FullNode, r cid.Cid) error
 	})
 }
 
-func printTipSet(format string, ts *types.TipSet) {
+func printTipSet(format string, ts *types.TipSet, afmt *AppFmt) {
 	format = strings.ReplaceAll(format, "<height>", fmt.Sprint(ts.Height()))
 	format = strings.ReplaceAll(format, "<time>", time.Unix(int64(ts.MinTimestamp()), 0).Format(time.Stamp))
 	blks := "[ "
@@ -897,7 +912,7 @@ func printTipSet(format string, ts *types.TipSet) {
 	format = strings.ReplaceAll(format, "<blocks>", blks)
 	format = strings.ReplaceAll(format, "<weight>", fmt.Sprint(ts.Blocks()[0].ParentWeight))
 
-	fmt.Println(format)
+	afmt.Println(format)
 }
 
 var ChainBisectCmd = &cli.Command{
@@ -918,6 +933,8 @@ var ChainBisectCmd = &cli.Command{
    For special path elements see 'chain get' help
 `,
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -961,7 +978,7 @@ var ChainBisectCmd = &cli.Command{
 			}
 
 			path := "/ipld/" + midTs.ParentState().String() + "/" + subPath
-			fmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)
+			afmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)
 
 			nd, err := api.ChainGetNode(ctx, path)
 			if err != nil {
@@ -988,32 +1005,32 @@ var ChainBisectCmd = &cli.Command{
 				if strings.TrimSpace(out.String()) != "false" {
 					end = mid
 					highest = midTs
-					fmt.Println("true")
+					afmt.Println("true")
 				} else {
 					start = mid
-					fmt.Printf("false (cli)\n")
+					afmt.Printf("false (cli)\n")
 				}
 			case *exec.ExitError:
 				if len(serr.String()) > 0 {
-					fmt.Println("error")
+					afmt.Println("error")
 
-					fmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
-					fmt.Println(string(b))
-					fmt.Println("<----")
+					afmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
+					afmt.Println(string(b))
+					afmt.Println("<----")
 					return xerrors.Errorf("error running bisect check: %s", serr.String())
 				}
 
 				start = mid
-				fmt.Println("false")
+				afmt.Println("false")
 			default:
 				return err
 			}
 
 			if start == end {
 				if strings.TrimSpace(out.String()) == "true" {
-					fmt.Println(midTs.Height())
+					afmt.Println(midTs.Height())
 				} else {
-					fmt.Println(prev)
+					afmt.Println(prev)
 				}
 				return nil
 			}
@@ -1058,7 +1075,7 @@ var ChainExportCmd = &cli.Command{
 			return fmt.Errorf("\"recent-stateroots\" has to be greater than %d", build.Finality)
 		}
 
-		fi, err := os.Create(cctx.Args().First())
+		fi, err := createExportFile(cctx.App, cctx.Args().First())
 		if err != nil {
 			return err
 		}
@@ -1118,6 +1135,8 @@ var SlashConsensusFault = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		srv, err := GetFullNodeServices(cctx)
 		if err != nil {
 			return err
@@ -1222,7 +1241,7 @@ var SlashConsensusFault = &cli.Command{
 			return err
 		}
 
-		fmt.Println(smsg.Cid())
+		afmt.Println(smsg.Cid())
 
 		return nil
 	},
@@ -1232,6 +1251,8 @@ var ChainGasPriceCmd = &cli.Command{
 	Name:  "gas-price",
 	Usage: "Estimate gas prices",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -1248,7 +1269,7 @@ var ChainGasPriceCmd = &cli.Command{
 				return err
 			}
 
-			fmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
+			afmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
 		}
 
 		return nil
@@ -1278,6 +1299,8 @@ var chainDecodeParamsCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -1329,7 +1352,7 @@ var chainDecodeParamsCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Println(pstr)
+		afmt.Println(pstr)
 
 		return nil
 	},
@@ -1362,6 +1385,8 @@ var chainEncodeParamsCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		if cctx.Args().Len() != 3 {
 			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
 		}
@@ -1410,9 +1435,9 @@ var chainEncodeParamsCmd = &cli.Command{
 
 		switch cctx.String("encoding") {
 		case "base64", "b64":
-			fmt.Println(base64.StdEncoding.EncodeToString(p))
+			afmt.Println(base64.StdEncoding.EncodeToString(p))
 		case "hex":
-			fmt.Println(hex.EncodeToString(p))
+			afmt.Println(hex.EncodeToString(p))
 		default:
 			return xerrors.Errorf("unknown encoding")
 		}
@@ -1420,3 +1445,16 @@ var chainEncodeParamsCmd = &cli.Command{
 		return nil
 	},
 }
+
+// createExportFile returns the export file handle from the app metadata, or creates a new file if it doesn't exist
+func createExportFile(app *cli.App, path string) (io.WriteCloser, error) {
+	if wc, ok := app.Metadata["export-file"]; ok {
+		return wc.(io.WriteCloser), nil
+	}
+
+	fi, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	return fi, nil
+}

cli/chain_test.go

@@ -0,0 +1,557 @@
+//stm: #cli
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/api"
+	types "github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin"
+	"github.com/golang/mock/gomock"
+	cid "github.com/ipfs/go-cid"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChainHead(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainHeadCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	gomock.InOrder(
+		mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+	)
+
+	//stm: @CLI_CHAIN_HEAD_001
+	err := app.Run([]string{"chain", "head"})
+	assert.NoError(t, err)
+
+	assert.Regexp(t, regexp.MustCompile(ts.Cids()[0].String()), buf.String())
+}
+
+func TestGetBlock(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetBlock))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	block := mock.MkBlock(nil, 0, 0)
+	blockMsgs := api.BlockMessages{}
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainGetBlock(ctx, block.Cid()).Return(block, nil),
+		mockApi.EXPECT().ChainGetBlockMessages(ctx, block.Cid()).Return(&blockMsgs, nil),
+		mockApi.EXPECT().ChainGetParentMessages(ctx, block.Cid()).Return([]api.Message{}, nil),
+		mockApi.EXPECT().ChainGetParentReceipts(ctx, block.Cid()).Return([]*types.MessageReceipt{}, nil),
+	)
+
+	//stm: @CLI_CHAIN_GET_BLOCK_001
+	err := app.Run([]string{"chain", "getblock", block.Cid().String()})
+	assert.NoError(t, err)
+
+	// expected output format
+	out := struct {
+		types.BlockHeader
+		BlsMessages    []*types.Message
+		SecpkMessages  []*types.SignedMessage
+		ParentReceipts []*types.MessageReceipt
+		ParentMessages []cid.Cid
+	}{}
+
+	err = json.Unmarshal(buf.Bytes(), &out)
+	assert.NoError(t, err)
+
+	assert.True(t, block.Cid().Equals(out.Cid()))
+}
+
+func TestReadOjb(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainReadObjCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	block := mock.MkBlock(nil, 0, 0)
+	obj := new(bytes.Buffer)
+	err := block.MarshalCBOR(obj)
+	assert.NoError(t, err)
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainReadObj(ctx, block.Cid()).Return(obj.Bytes(), nil),
+	)
+
+	//stm: @CLI_CHAIN_READ_OBJECT_001
+	err = app.Run([]string{"chain", "read-obj", block.Cid().String()})
+	assert.NoError(t, err)
+
+	assert.Equal(t, buf.String(), fmt.Sprintf("%x\n", obj.Bytes()))
+}
+
+func TestChainDeleteObj(t *testing.T) {
+	cmd := WithCategory("chain", ChainDeleteObjCmd)
+	block := mock.MkBlock(nil, 0, 0)
+
+	// given no force flag, it should return an error and no API calls should be made
+	t.Run("no-really-do-it", func(t *testing.T) {
+		app, _, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		//stm: @CLI_CHAIN_DELETE_OBJECT_002
+		err := app.Run([]string{"chain", "delete-obj", block.Cid().String()})
+		assert.Error(t, err)
+	})
+
+	// given a force flag, it calls API delete
+	t.Run("really-do-it", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainDeleteObj(ctx, block.Cid()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_DELETE_OBJECT_001
+		err := app.Run([]string{"chain", "delete-obj", "--really-do-it=true", block.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), block.Cid().String())
+	})
+}
+
+func TestChainStatObj(t *testing.T) {
+	cmd := WithCategory("chain", ChainStatObjCmd)
+	block := mock.MkBlock(nil, 0, 0)
+	stat := api.ObjStat{Size: 123, Links: 321}
+
+	checkOutput := func(buf *bytes.Buffer) {
+		out := buf.String()
+		outSplit := strings.Split(out, "\n")
+
+		assert.Contains(t, outSplit[0], fmt.Sprintf("%d", stat.Links))
+		assert.Contains(t, outSplit[1], fmt.Sprintf("%d", stat.Size))
+	}
+
+	// given no --base flag, it calls ChainStatObj with base=cid.Undef
+	t.Run("no-base", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), cid.Undef).Return(stat, nil),
+		)
+
+		//stm: @CLI_CHAIN_STAT_OBJECT_001
+		err := app.Run([]string{"chain", "stat-obj", block.Cid().String()})
+		assert.NoError(t, err)
+
+		checkOutput(buf)
+	})
+
+	// given a --base flag, it calls ChainStatObj with that base
+	t.Run("base", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), block.Cid()).Return(stat, nil),
+		)
+
+		//stm: @CLI_CHAIN_STAT_OBJECT_002
+		err := app.Run([]string{"chain", "stat-obj", fmt.Sprintf("-base=%s", block.Cid().String()), block.Cid().String()})
+		assert.NoError(t, err)
+
+		checkOutput(buf)
+	})
+}
+
+func TestChainGetMsg(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetMsgCmd))
+	defer done()
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+
+	obj := new(bytes.Buffer)
+	err = msg.MarshalCBOR(obj)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainReadObj(ctx, msg.Cid()).Return(obj.Bytes(), nil),
+	)
+
+	//stm: @CLI_CHAIN_GET_MESSAGE_001
+	err = app.Run([]string{"chain", "getmessage", msg.Cid().String()})
+	assert.NoError(t, err)
+
+	var out types.Message
+	err = json.Unmarshal(buf.Bytes(), &out)
+	assert.NoError(t, err)
+
+	assert.Equal(t, *msg, out)
+}
+
+func TestSetHead(t *testing.T) {
+	cmd := WithCategory("chain", ChainSetHeadCmd)
+	genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	ts := mock.TipSet(mock.MkBlock(genesis, 1, 0))
+	epoch := abi.ChainEpoch(uint64(0))
+
+	// given the -genesis flag, resets head to genesis ignoring the provided ts positional argument
+	t.Run("genesis", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetGenesis(ctx).Return(genesis, nil),
+			mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_003
+		err := app.Run([]string{"chain", "sethead", "-genesis=true", ts.Key().String()})
+		assert.NoError(t, err)
+	})
+
+	// given the -epoch flag, resets head to given epoch, ignoring the provided ts positional argument
+	t.Run("epoch", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(genesis, nil),
+			mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_002
+		err := app.Run([]string{"chain", "sethead", fmt.Sprintf("-epoch=%s", epoch), ts.Key().String()})
+		assert.NoError(t, err)
+	})
+
+	// given no flag, resets the head to given tipset key
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetBlock(ctx, ts.Key().Cids()[0]).Return(ts.Blocks()[0], nil),
+			mockApi.EXPECT().ChainSetHead(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_001
+		err := app.Run([]string{"chain", "sethead", ts.Key().Cids()[0].String()})
+		assert.NoError(t, err)
+	})
+}
+
+func TestInspectUsage(t *testing.T) {
+	cmd := WithCategory("chain", ChainInspectUsage)
+	ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+	msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
+
+	actor := &types.Actor{
+		Code:    builtin.StorageMarketActorCodeID,
+		Nonce:   0,
+		Balance: big.NewInt(1000000000),
+	}
+
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+			mockApi.EXPECT().ChainGetParentMessages(ctx, ts.Blocks()[0].Cid()).Return(msgs, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, ts.Parents()).Return(nil, nil),
+			mockApi.EXPECT().StateGetActor(ctx, *to, ts.Key()).Return(actor, nil),
+		)
+
+		//stm: @CLI_CHAIN_INSPECT_USAGE_001
+		err := app.Run([]string{"chain", "inspect-usage"})
+		assert.NoError(t, err)
+
+		out := buf.String()
+
+		// output is plaintext, had to do string matching
+		assert.Contains(t, out, from.String())
+		assert.Contains(t, out, to.String())
+		// check for gas by sender
+		assert.Contains(t, out, "By Sender")
+		// check for gas by method
+		assert.Contains(t, out, "By Method:\nSend")
+	})
+}
+
+func TestChainList(t *testing.T) {
+	cmd := WithCategory("chain", ChainListCmd)
+	genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	blk := mock.MkBlock(genesis, 0, 0)
+	blk.Height = 1
+	head := mock.TipSet(blk)
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+	msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
+	blockMsgs := &api.BlockMessages{}
+	receipts := []*types.MessageReceipt{}
+
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// same method gets called mocked multiple times bcs it's called in a for loop for all tipsets (2 in this case)
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(genesis, nil),
+			mockApi.EXPECT().ChainGetBlockMessages(ctx, genesis.Blocks()[0].Cid()).Return(blockMsgs, nil),
+			mockApi.EXPECT().ChainGetParentMessages(ctx, head.Blocks()[0].Cid()).Return(msgs, nil),
+			mockApi.EXPECT().ChainGetParentReceipts(ctx, head.Blocks()[0].Cid()).Return(receipts, nil),
+			mockApi.EXPECT().ChainGetBlockMessages(ctx, head.Blocks()[0].Cid()).Return(blockMsgs, nil),
+		)
+
+		//stm: CLI_CHAIN_LIST_001
+		err := app.Run([]string{"chain", "love", "--gas-stats=true"}) // chain is love ❤️
+		assert.NoError(t, err)
+
+		out := buf.String()
+
+		// should print out 2 blocks, indexed with 0: and 1:
+		assert.Contains(t, out, "0:")
+		assert.Contains(t, out, "1:")
+	})
+}
+
+func TestChainGet(t *testing.T) {
+	blk := mock.MkBlock(nil, 0, 0)
+	ts := mock.TipSet(blk)
+	cmd := WithCategory("chain", ChainGetCmd)
+
+	// given no -as-type flag & ipfs prefix, should print object as JSON if it's marshalable
+	t.Run("ipfs", func(t *testing.T) {
+		path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_001
+		err := app.Run([]string{"chain", "get", path})
+		assert.NoError(t, err)
+
+		var out types.BlockHeader
+		err = json.Unmarshal(buf.Bytes(), &out)
+		assert.NoError(t, err)
+		assert.Equal(t, *blk, out)
+	})
+
+	// given no -as-type flag & ipfs prefix, should traverse from head.ParentStateRoot and print JSON if it's marshalable
+	t.Run("pstate", func(t *testing.T) {
+		p1 := "/pstate"
+		p2 := fmt.Sprintf("/ipfs/%s", ts.ParentState().String())
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+			mockApi.EXPECT().ChainGetNode(ctx, p2).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_002
+		err := app.Run([]string{"chain", "get", p1})
+		assert.NoError(t, err)
+
+		var out types.BlockHeader
+		err = json.Unmarshal(buf.Bytes(), &out)
+		assert.NoError(t, err)
+		assert.Equal(t, *blk, out)
+	})
+
+	// given an unknown -as-type value, return an error
+	t.Run("unknown-type", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_004
+		err := app.Run([]string{"chain", "get", "-as-type=foo", path})
+		assert.Error(t, err)
+	})
+}
+
+func TestChainBisect(t *testing.T) {
+	blk1 := mock.MkBlock(nil, 0, 0)
+	blk1.Height = 0
+	ts1 := mock.TipSet(blk1)
+
+	blk2 := mock.MkBlock(ts1, 0, 0)
+	blk2.Height = 1
+	ts2 := mock.TipSet(blk2)
+
+	subpath := "whatever/its/mocked"
+	minHeight := uint64(0)
+	maxHeight := uint64(1)
+	shell := "echo"
+
+	path := fmt.Sprintf("/ipld/%s/%s", ts2.ParentState(), subpath)
+
+	cmd := WithCategory("chain", ChainBisectCmd)
+
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), types.EmptyTSK).Return(ts2, nil),
+		mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), ts2.Key()).Return(ts2, nil),
+		mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk2.Cid(), Obj: blk2}, nil),
+	)
+
+	//stm: @CLI_CHAIN_BISECT_001
+	err := app.Run([]string{"chain", "bisect", fmt.Sprintf("%d", minHeight), fmt.Sprintf("%d", maxHeight), subpath, shell})
+	assert.NoError(t, err)
+
+	out := buf.String()
+	assert.Contains(t, out, path)
+}
+
+func TestChainExport(t *testing.T) {
+	app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainExportCmd))
+	defer done()
+
+	// export writes to a file, I mocked it so there are no side-effects
+	mockFile := mockExportFile{new(bytes.Buffer)}
+	app.Metadata["export-file"] = mockFile
+
+	blk := mock.MkBlock(nil, 0, 0)
+	ts := mock.TipSet(blk)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	export := make(chan []byte, 2)
+	expBytes := []byte("whatever")
+	export <- expBytes
+	export <- []byte{} // empty slice means export is complete
+	close(export)
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+		mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key()).Return(export, nil),
+	)
+
+	//stm: @CLI_CHAIN_EXPORT_001
+	err := app.Run([]string{"chain", "export", "whatever.car"})
+	assert.NoError(t, err)
+
+	assert.Equal(t, expBytes, mockFile.Bytes())
+}
+
+func TestChainGasPrice(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGasPriceCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// estimate gas is called with various num blocks in implementation,
+	// so we mock and count how many times it's called, and we expect that many results printed
+	calls := 0
+	mockApi.
+		EXPECT().
+		GasEstimateGasPremium(ctx, gomock.Any(), builtin.SystemActorAddr, int64(10000), types.EmptyTSK).
+		Return(big.NewInt(0), nil).
+		AnyTimes().
+		Do(func(a, b, c, d, e interface{}) { // looks funny, but we don't care about args here, just counting
+			calls++
+		})
+
+	//stm: @CLI_CHAIN_GAS_PRICE_001
+	err := app.Run([]string{"chain", "gas-price"})
+	assert.NoError(t, err)
+
+	lines := strings.Split(strings.Trim(buf.String(), "\n"), "\n")
+	assert.Equal(t, calls, len(lines))
+}
+
+type mockExportFile struct {
+	*bytes.Buffer
+}
+
+func (mef mockExportFile) Close() error {
+	return nil
+}

cli/client.go

@@ -667,6 +667,8 @@ uiLoop:
 
 			state = "miner"
 		case "miner":
+			maddrs = maddrs[:0]
+			ask = ask[:0]
 			afmt.Print("Miner Addresses (f0.. f0..), none to find: ")
 
 			_maddrsStr, _, err := rl.ReadLine()
@@ -802,7 +804,8 @@ uiLoop:
 
 			dealCount, err = strconv.ParseInt(string(dealcStr), 10, 64)
 			if err != nil {
-				return err
+				printErr(xerrors.Errorf("reading deal count: invalid number"))
+				continue
 			}
 
 			color.Blue(".. Picking miners")
@@ -859,12 +862,13 @@ uiLoop:
 
 				a, err := api.ClientQueryAsk(ctx, *mi.PeerId, maddr)
 				if err != nil {
-					printErr(xerrors.Errorf("failed to query ask: %w", err))
+					printErr(xerrors.Errorf("failed to query ask for miner %s: %w", maddr.String(), err))
 					state = "miner"
 					continue uiLoop
 				}
 
 				ask = append(ask, *a)
+
 			}
 
 			// TODO: run more validation

cli/filplus.go

@@ -1,7 +1,9 @@
 package cli
 
 import (
+	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"
 
 	verifreg4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg"
@@ -34,6 +36,7 @@ var filplusCmd = &cli.Command{
 		filplusListClientsCmd,
 		filplusCheckClientCmd,
 		filplusCheckNotaryCmd,
+		filplusSignRemoveDataCapProposal,
 	},
 }
 
@@ -274,3 +277,112 @@ func checkNotary(ctx context.Context, api v0api.FullNode, vaddr address.Address)
 
 	return st.VerifierDataCap(vid)
 }
+
+var filplusSignRemoveDataCapProposal = &cli.Command{
+	Name:  "sign-remove-data-cap-proposal",
+	Usage: "allows a notary to sign a Remove Data Cap Proposal",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name:     "id",
+			Usage:    "specify the RemoveDataCapProposal ID (will look up on chain if unspecified)",
+			Required: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 3 {
+			return fmt.Errorf("must specify three arguments: notary address, client address, and allowance to remove")
+		}
+
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("failed to get full node api: %w", err)
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		act, err := api.StateGetActor(ctx, verifreg.Address, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("failed to get verifreg actor: %w", err)
+		}
+
+		apibs := blockstore.NewAPIBlockstore(api)
+		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
+
+		st, err := verifreg.Load(store, act)
+		if err != nil {
+			return xerrors.Errorf("failed to load verified registry state: %w", err)
+		}
+
+		verifier, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+		verifierIdAddr, err := api.StateLookupID(ctx, verifier, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		client, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+		clientIdAddr, err := api.StateLookupID(ctx, client, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		allowanceToRemove, err := types.BigFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		_, dataCap, err := st.VerifiedClientDataCap(clientIdAddr)
+		if err != nil {
+			return xerrors.Errorf("failed to find verified client data cap: %w", err)
+		}
+		if dataCap.LessThanEqual(big.Zero()) {
+			return xerrors.Errorf("client data cap %s is less than amount requested to be removed %s", dataCap.String(), allowanceToRemove.String())
+		}
+
+		found, _, err := checkNotary(ctx, api, verifier)
+		if err != nil {
+			return xerrors.Errorf("failed to check notary status: %w", err)
+		}
+
+		if !found {
+			return xerrors.New("verifier address must be a notary")
+		}
+
+		id := cctx.Uint64("id")
+		if id == 0 {
+			_, id, err = st.RemoveDataCapProposalID(verifierIdAddr, clientIdAddr)
+			if err != nil {
+				return xerrors.Errorf("failed find remove data cap proposal id: %w", err)
+			}
+		}
+
+		params := verifreg.RemoveDataCapProposal{
+			RemovalProposalID: verifreg.RmDcProposalID{ProposalID: id},
+			DataCapAmount:     allowanceToRemove,
+			VerifiedClient:    clientIdAddr,
+		}
+
+		paramBuf := new(bytes.Buffer)
+		paramBuf.WriteString(verifreg.SignatureDomainSeparation_RemoveDataCap)
+		err = params.MarshalCBOR(paramBuf)
+		if err != nil {
+			return xerrors.Errorf("failed to marshall paramBuf: %w", err)
+		}
+
+		sig, err := api.WalletSign(ctx, verifier, paramBuf.Bytes())
+		if err != nil {
+			return xerrors.Errorf("failed to sign message: %w", err)
+		}
+
+		sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)
+
+		fmt.Println(hex.EncodeToString(sigBytes))
+
+		return nil
+	},
+}

cli/mocks_test.go

@@ -0,0 +1,32 @@
+package cli
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/mocks"
+	"github.com/golang/mock/gomock"
+	ucli "github.com/urfave/cli/v2"
+)
+
+// newMockAppWithFullAPI returns a gomock-ed CLI app used for unit tests
+// see cli/util/api.go:GetFullNodeAPI for mock API injection
+func NewMockAppWithFullAPI(t *testing.T, cmd *ucli.Command) (*ucli.App, *mocks.MockFullNode, *bytes.Buffer, func()) {
+	app := ucli.NewApp()
+	app.Commands = ucli.Commands{cmd}
+	app.Setup()
+
+	// create and inject the mock API into app Metadata
+	ctrl := gomock.NewController(t)
+	mockFullNode := mocks.NewMockFullNode(ctrl)
+	var fullNode api.FullNode = mockFullNode
+	app.Metadata["test-full-api"] = fullNode
+
+	// this will only work if the implementation uses the app.Writer,
+	// if it uses fmt.*, it has to be refactored
+	buf := &bytes.Buffer{}
+	app.Writer = buf
+
+	return app, mockFullNode, buf, ctrl.Finish
+}

cli/mpool.go

@@ -60,6 +60,8 @@ var MpoolPending = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -72,7 +74,7 @@ var MpoolPending = &cli.Command{
 		if tos := cctx.String("to"); tos != "" {
 			a, err := address.NewFromString(tos)
 			if err != nil {
-				return fmt.Errorf("given 'to' address %q was invalid: %w", tos, err)
+				return xerrors.Errorf("given 'to' address %q was invalid: %w", tos, err)
 			}
 			toa = a
 		}
@@ -80,7 +82,7 @@ var MpoolPending = &cli.Command{
 		if froms := cctx.String("from"); froms != "" {
 			a, err := address.NewFromString(froms)
 			if err != nil {
-				return fmt.Errorf("given 'from' address %q was invalid: %w", froms, err)
+				return xerrors.Errorf("given 'from' address %q was invalid: %w", froms, err)
 			}
 			froma = a
 		}
@@ -119,13 +121,13 @@ var MpoolPending = &cli.Command{
 			}
 
 			if cctx.Bool("cids") {
-				fmt.Println(msg.Cid())
+				afmt.Println(msg.Cid())
 			} else {
 				out, err := json.MarshalIndent(msg, "", "  ")
 				if err != nil {
 					return err
 				}
-				fmt.Println(string(out))
+				afmt.Println(string(out))
 			}
 		}
 
@@ -216,6 +218,8 @@ var MpoolStat = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -234,6 +238,7 @@ var MpoolStat = &cli.Command{
 			currTs := ts
 			for i := 0; i < cctx.Int("basefee-lookback"); i++ {
 				currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
+
 				if err != nil {
 					return xerrors.Errorf("walking chain: %w", err)
 				}
@@ -296,7 +301,7 @@ var MpoolStat = &cli.Command{
 		for a, bkt := range buckets {
 			act, err := api.StateGetActor(ctx, a, ts.Key())
 			if err != nil {
-				fmt.Printf("%s, err: %s\n", a, err)
+				afmt.Printf("%s, err: %s\n", a, err)
 				continue
 			}
 
@@ -350,11 +355,11 @@ var MpoolStat = &cli.Command{
 			total.belowPast += stat.belowPast
 			total.gasLimit = big.Add(total.gasLimit, stat.gasLimit)
 
-			fmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
+			afmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
 		}
 
-		fmt.Println("-----")
-		fmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)
+		afmt.Println("-----")
+		afmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)
 
 		return nil
 	},
@@ -385,8 +390,9 @@ var MpoolReplaceCmd = &cli.Command{
 			Usage: "Spend up to X FIL for this message in units of FIL. Previously when flag was `max-fee` units were in attoFIL. Applicable for auto mode",
 		},
 	},
-	ArgsUsage: "<from nonce> | <message-cid>",
+	ArgsUsage: "<from> <nonce> | <message-cid>",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
@@ -407,13 +413,14 @@ var MpoolReplaceCmd = &cli.Command{
 
 			msg, err := api.ChainGetMessage(ctx, mcid)
 			if err != nil {
-				return fmt.Errorf("could not find referenced message: %w", err)
+				return xerrors.Errorf("could not find referenced message: %w", err)
 			}
 
 			from = msg.From
 			nonce = msg.Nonce
 		case 2:
-			f, err := address.NewFromString(cctx.Args().Get(0))
+			arg0 := cctx.Args().Get(0)
+			f, err := address.NewFromString(arg0)
 			if err != nil {
 				return err
 			}
@@ -448,7 +455,7 @@ var MpoolReplaceCmd = &cli.Command{
 		}
 
 		if found == nil {
-			return fmt.Errorf("no pending message found from %s with nonce %d", from, nonce)
+			return xerrors.Errorf("no pending message found from %s with nonce %d", from, nonce)
 		}
 
 		msg := found.Message
@@ -460,7 +467,7 @@ var MpoolReplaceCmd = &cli.Command{
 			if cctx.IsSet("fee-limit") {
 				maxFee, err := types.ParseFIL(cctx.String("fee-limit"))
 				if err != nil {
-					return fmt.Errorf("parsing max-spend: %w", err)
+					return xerrors.Errorf("parsing max-spend: %w", err)
 				}
 				mss = &lapi.MessageSendSpec{
 					MaxFee: abi.TokenAmount(maxFee),
@@ -472,7 +479,7 @@ var MpoolReplaceCmd = &cli.Command{
 			msg.GasPremium = abi.NewTokenAmount(0)
 			retm, err := api.GasEstimateMessageGas(ctx, &msg, mss, types.EmptyTSK)
 			if err != nil {
-				return fmt.Errorf("failed to estimate gas values: %w", err)
+				return xerrors.Errorf("failed to estimate gas values: %w", err)
 			}
 
 			msg.GasPremium = big.Max(retm.GasPremium, minRBF)
@@ -489,26 +496,26 @@ var MpoolReplaceCmd = &cli.Command{
 			}
 			msg.GasPremium, err = types.BigFromString(cctx.String("gas-premium"))
 			if err != nil {
-				return fmt.Errorf("parsing gas-premium: %w", err)
+				return xerrors.Errorf("parsing gas-premium: %w", err)
 			}
 			// TODO: estimate fee cap here
 			msg.GasFeeCap, err = types.BigFromString(cctx.String("gas-feecap"))
 			if err != nil {
-				return fmt.Errorf("parsing gas-feecap: %w", err)
+				return xerrors.Errorf("parsing gas-feecap: %w", err)
 			}
 		}
 
 		smsg, err := api.WalletSignMessage(ctx, msg.From, &msg)
 		if err != nil {
-			return fmt.Errorf("failed to sign message: %w", err)
+			return xerrors.Errorf("failed to sign message: %w", err)
 		}
 
 		cid, err := api.MpoolPush(ctx, smsg)
 		if err != nil {
-			return fmt.Errorf("failed to push new message to mempool: %w", err)
+			return xerrors.Errorf("failed to push new message to mempool: %w", err)
 		}
 
-		fmt.Println("new message cid: ", cid)
+		afmt.Println("new message cid: ", cid)
 		return nil
 	},
 }
@@ -531,6 +538,8 @@ var MpoolFindCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -548,7 +557,7 @@ var MpoolFindCmd = &cli.Command{
 		if cctx.IsSet("to") {
 			a, err := address.NewFromString(cctx.String("to"))
 			if err != nil {
-				return fmt.Errorf("'to' address was invalid: %w", err)
+				return xerrors.Errorf("'to' address was invalid: %w", err)
 			}
 
 			toFilter = a
@@ -557,7 +566,7 @@ var MpoolFindCmd = &cli.Command{
 		if cctx.IsSet("from") {
 			a, err := address.NewFromString(cctx.String("from"))
 			if err != nil {
-				return fmt.Errorf("'from' address was invalid: %w", err)
+				return xerrors.Errorf("'from' address was invalid: %w", err)
 			}
 
 			fromFilter = a
@@ -591,7 +600,7 @@ var MpoolFindCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Println(string(b))
+		afmt.Println(string(b))
 		return nil
 	},
 }
@@ -605,6 +614,8 @@ var MpoolConfig = &cli.Command{
 			return cli.ShowCommandHelp(cctx, cctx.Command.Name)
 		}
 
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -624,7 +635,7 @@ var MpoolConfig = &cli.Command{
 				return err
 			}
 
-			fmt.Println(string(bytes))
+			afmt.Println(string(bytes))
 		} else {
 			cfg := new(types.MpoolConfig)
 			bytes := []byte(cctx.Args().Get(0))
@@ -651,6 +662,8 @@ var MpoolGasPerfCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -717,7 +730,7 @@ var MpoolGasPerfCmd = &cli.Command{
 			gasReward := getGasReward(m)
 			gasPerf := getGasPerf(gasReward, m.Message.GasLimit)
 
-			fmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
+			afmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
 		}
 
 		return nil

cli/mpool_test.go

@@ -0,0 +1,582 @@
+//stm: #cli
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"encoding/json"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStat(t *testing.T) {
+
+	t.Run("local", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// mock actor to return for the sender
+		actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr, toAddr}, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_STAT_002
+		err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1", "--local"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "Nonce past: 1")
+	})
+
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// mock actor to return for the sender
+		actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_STAT_001
+		err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "Nonce past: 1")
+	})
+}
+
+func TestPending(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_001
+		err = app.Run([]string{"mpool", "pending", "--cids"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("local", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_002
+		err = app.Run([]string{"mpool", "pending", "--local"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("to", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_003
+		err = app.Run([]string{"mpool", "pending", "--to", sm.Message.To.String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("from", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_004
+		err = app.Run([]string{"mpool", "pending", "--from", sm.Message.From.String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+}
+
+func TestReplace(t *testing.T) {
+	t.Run("manual", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, &sm.Message).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_002
+		err = app.Run([]string{"mpool", "replace", "--gas-premium", "1", "--gas-feecap", "100", sm.Cid().String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("auto", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// gas fee param should be equal to the one passed in the cli invocation (used below)
+		maxFee := "1000000"
+		parsedFee, err := types.ParseFIL(maxFee)
+		if err != nil {
+			t.Fatal(err)
+		}
+		mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			// use gomock.any to match the message in expected api calls
+			// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
+			mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_002
+		err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Cid().String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("sender / nonce", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// gas fee param should be equal to the one passed in the cli invocation (used below)
+		maxFee := "1000000"
+		parsedFee, err := types.ParseFIL(maxFee)
+		if err != nil {
+			t.Fatal(err)
+		}
+		mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			// use gomock.any to match the message in expected api calls
+			// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
+			mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_001
+		err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Message.From.String(), fmt.Sprint(sm.Message.Nonce)})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+}
+
+func TestFindMsg(t *testing.T) {
+	t.Run("from", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_001
+		err = app.Run([]string{"mpool", "find", "--from", sm.Message.From.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("to", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_002
+		err = app.Run([]string{"mpool", "find", "--to", sm.Message.To.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("method", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_003
+		err = app.Run([]string{"mpool", "find", "--method", sm.Message.Method.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+}
+
+func TestGasPerf(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 13, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_GAS_PERF_002
+		err = app.Run([]string{"mpool", "gas-perf", "--all", "true"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Message.From.String())
+		assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
+	})
+
+	t.Run("local", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 13, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_GAS_PERF_001
+		err = app.Run([]string{"mpool", "gas-perf"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Message.From.String())
+		assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
+	})
+}
+
+func TestConfig(t *testing.T) {
+	t.Run("get", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25}
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_CONFIG_001
+		err = app.Run([]string{"mpool", "config"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), mpoolCfg.PriorityAddrs[0].String())
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitHigh))
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitLow))
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.ReplaceByFeeRatio))
+	})
+
+	t.Run("set", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33}
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil),
+		)
+
+		bytes, err := json.Marshal(mpoolCfg)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		//stm: @CLI_MEMPOOL_CONFIG_002
+		err = app.Run([]string{"mpool", "config", string(bytes)})
+		assert.NoError(t, err)
+	})
+}

cli/net.go

@@ -38,6 +38,9 @@ var NetCmd = &cli.Command{
 		NetBlockCmd,
 		NetStatCmd,
 		NetLimitCmd,
+		NetProtectAdd,
+		NetProtectRemove,
+		NetProtectList,
 	},
 }
 
@@ -708,3 +711,108 @@ var NetLimitCmd = &cli.Command{
 		return enc.Encode(result)
 	},
 }
+
+var NetProtectAdd = &cli.Command{
+	Name:      "protect",
+	Usage:     "Add one or more peer IDs to the list of protected peer connections",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectAdd(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("added to protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+var NetProtectRemove = &cli.Command{
+	Name:      "unprotect",
+	Usage:     "Remove one or more peer IDs from the list of protected peer connections.",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectRemove(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("removed from protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+// decodePeerIDsFromArgs decodes all the arguments present in cli.Context.Args as peer.ID.
+//
+// This function requires at least one argument to be present, and arguments must not be empty
+// string. Otherwise, an error is returned.
+func decodePeerIDsFromArgs(cctx *cli.Context) ([]peer.ID, error) {
+	pidArgs := cctx.Args().Slice()
+	if len(pidArgs) == 0 {
+		return nil, xerrors.Errorf("must specify at least one peer ID as an argument")
+	}
+	var pids []peer.ID
+	for _, pidStr := range pidArgs {
+		if pidStr == "" {
+			return nil, xerrors.Errorf("peer ID must not be empty")
+		}
+		pid, err := peer.Decode(pidStr)
+		if err != nil {
+			return nil, err
+		}
+		pids = append(pids, pid)
+	}
+	return pids, nil
+}
+
+var NetProtectList = &cli.Command{
+	Name:  "list-protected",
+	Usage: "List the peer IDs with protected connection.",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		pids, err := api.NetProtectList(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, pid := range pids {
+			fmt.Printf("%s\n", pid)
+		}
+		return nil
+	},
+}

cli/sync.go

@@ -33,6 +33,8 @@ var SyncStatusCmd = &cli.Command{
 	Name:  "status",
 	Usage: "check sync status",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		apic, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -45,9 +47,9 @@ var SyncStatusCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Println("sync status:")
+		afmt.Println("sync status:")
 		for _, ss := range state.ActiveSyncs {
-			fmt.Printf("worker %d:\n", ss.WorkerID)
+			afmt.Printf("worker %d:\n", ss.WorkerID)
 			var base, target []cid.Cid
 			var heightDiff int64
 			var theight abi.ChainEpoch
@@ -62,20 +64,20 @@ var SyncStatusCmd = &cli.Command{
 			} else {
 				heightDiff = 0
 			}
-			fmt.Printf("\tBase:\t%s\n", base)
-			fmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
-			fmt.Printf("\tHeight diff:\t%d\n", heightDiff)
-			fmt.Printf("\tStage: %s\n", ss.Stage)
-			fmt.Printf("\tHeight: %d\n", ss.Height)
+			afmt.Printf("\tBase:\t%s\n", base)
+			afmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
+			afmt.Printf("\tHeight diff:\t%d\n", heightDiff)
+			afmt.Printf("\tStage: %s\n", ss.Stage)
+			afmt.Printf("\tHeight: %d\n", ss.Height)
 			if ss.End.IsZero() {
 				if !ss.Start.IsZero() {
-					fmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
+					afmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
 				}
 			} else {
-				fmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
+				afmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
 			}
 			if ss.Stage == api.StageSyncErrored {
-				fmt.Printf("\tError: %s\n", ss.Message)
+				afmt.Printf("\tError: %s\n", ss.Message)
 			}
 		}
 		return nil
@@ -168,6 +170,8 @@ var SyncCheckBadCmd = &cli.Command{
 	Usage:     "check if the given block was marked bad, and for what reason",
 	ArgsUsage: "[blockCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		napi, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -190,11 +194,11 @@ var SyncCheckBadCmd = &cli.Command{
 		}
 
 		if reason == "" {
-			fmt.Println("block was not marked as bad")
+			afmt.Println("block was not marked as bad")
 			return nil
 		}
 
-		fmt.Println(reason)
+		afmt.Println(reason)
 		return nil
 	},
 }

cli/sync_test.go

@@ -0,0 +1,189 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSyncStatus(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncStatusCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ts1 := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	ts2 := mock.TipSet(mock.MkBlock(ts1, 0, 0))
+
+	start := time.Now()
+	end := start.Add(time.Minute)
+
+	state := &api.SyncState{
+		ActiveSyncs: []api.ActiveSync{{
+			WorkerID: 1,
+			Base:     ts1,
+			Target:   ts2,
+			Stage:    api.StageMessages,
+			Height:   abi.ChainEpoch(0),
+			Start:    start,
+			End:      end,
+			Message:  "whatever",
+		}},
+		VMApplied: 0,
+	}
+
+	mockApi.EXPECT().SyncState(ctx).Return(state, nil)
+
+	//stm: @CLI_SYNC_STATUS_001
+	err := app.Run([]string{"sync", "status"})
+	assert.NoError(t, err)
+
+	out := buf.String()
+
+	// output is plaintext, had to do string matching
+	assert.Contains(t, out, fmt.Sprintf("Base:\t[%s]", ts1.Blocks()[0].Cid().String()))
+	assert.Contains(t, out, fmt.Sprintf("Target:\t[%s]", ts2.Blocks()[0].Cid().String()))
+	assert.Contains(t, out, "Height diff:\t1")
+	assert.Contains(t, out, "Stage: message sync")
+	assert.Contains(t, out, "Height: 0")
+	assert.Contains(t, out, "Elapsed: 1m0s")
+}
+
+func TestSyncMarkBad(t *testing.T) {
+	app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncMarkBadCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	blk := mock.MkBlock(nil, 0, 0)
+
+	mockApi.EXPECT().SyncMarkBad(ctx, blk.Cid()).Return(nil)
+
+	//stm: @CLI_SYNC_MARK_BAD_001
+	err := app.Run([]string{"sync", "mark-bad", blk.Cid().String()})
+	assert.NoError(t, err)
+}
+
+func TestSyncUnmarkBad(t *testing.T) {
+	t.Run("one-block", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+
+		mockApi.EXPECT().SyncUnmarkBad(ctx, blk.Cid()).Return(nil)
+
+		//stm: @CLI_SYNC_UNMARK_BAD_001
+		err := app.Run([]string{"sync", "unmark-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+	})
+
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		mockApi.EXPECT().SyncUnmarkAllBad(ctx).Return(nil)
+
+		//stm: @CLI_SYNC_UNMARK_BAD_002
+		err := app.Run([]string{"sync", "unmark-bad", "-all"})
+		assert.NoError(t, err)
+	})
+}
+
+func TestSyncCheckBad(t *testing.T) {
+	t.Run("not-bad", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+
+		mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return("", nil)
+
+		//stm: @CLI_SYNC_CHECK_BAD_002
+		err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "block was not marked as bad")
+	})
+
+	t.Run("bad", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+		reason := "whatever"
+
+		mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return(reason, nil)
+
+		//stm: @CLI_SYNC_CHECK_BAD_001
+		err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), reason)
+	})
+}
+
+func TestSyncCheckpoint(t *testing.T) {
+	t.Run("tipset", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+		ts := mock.TipSet(blk)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetBlock(ctx, blk.Cid()).Return(blk, nil),
+			mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_SYNC_CHECKPOINT_001
+		err := app.Run([]string{"sync", "checkpoint", blk.Cid().String()})
+		assert.NoError(t, err)
+	})
+
+	t.Run("epoch", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		epoch := abi.ChainEpoch(0)
+		blk := mock.MkBlock(nil, 0, 0)
+		ts := mock.TipSet(blk)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(ts, nil),
+			mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_SYNC_CHECKPOINT_002
+		err := app.Run([]string{"sync", "checkpoint", fmt.Sprintf("-epoch=%d", epoch)})
+		assert.NoError(t, err)
+	})
+}

cli/util/api.go

@@ -223,6 +223,11 @@ func GetCommonAPI(ctx *cli.Context) (api.CommonNet, jsonrpc.ClientCloser, error)
 }
 
 func GetFullNodeAPI(ctx *cli.Context) (v0api.FullNode, jsonrpc.ClientCloser, error) {
+	// use the mocked API in CLI unit tests, see cli/mocks_test.go for mock definition
+	if mock, ok := ctx.App.Metadata["test-full-api"]; ok {
+		return &v0api.WrapperV1Full{FullNode: mock.(v1api.FullNode)}, func() {}, nil
+	}
+
 	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
 		return &v0api.WrapperV1Full{FullNode: tn.(v1api.FullNode)}, func() {}, nil
 	}

cmd/lotus-bench/main.go

@@ -276,6 +276,13 @@ var sealBenchCmd = &cli.Command{
 			if err != nil {
 				return xerrors.Errorf("failed to run seals: %w", err)
 			}
+			for _, s := range extendedSealedSectors {
+				sealedSectors = append(sealedSectors, proof.SectorInfo{
+					SealedCID:    s.SealedCID,
+					SectorNumber: s.SectorNumber,
+					SealProof:    s.SealProof,
+				})
+			}
 		} else {
 			// TODO: implement sbfs.List() and use that for all cases (preexisting sectorbuilder or not)
 

cmd/lotus-miner/dagstore.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/fatih/color"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/lotus/api"
@@ -21,6 +22,7 @@ var dagstoreCmd = &cli.Command{
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
 		dagstoreGcCmd,
+		dagstoreLookupPiecesCmd,
 	},
 }
 
@@ -52,38 +54,7 @@ var dagstoreListShardsCmd = &cli.Command{
 			return err
 		}
 
-		if len(shards) == 0 {
-			return nil
-		}
-
-		tw := tablewriter.New(
-			tablewriter.Col("Key"),
-			tablewriter.Col("State"),
-			tablewriter.Col("Error"),
-		)
-
-		colors := map[string]color.Attribute{
-			"ShardStateAvailable": color.FgGreen,
-			"ShardStateServing":   color.FgBlue,
-			"ShardStateErrored":   color.FgRed,
-			"ShardStateNew":       color.FgYellow,
-		}
-
-		for _, s := range shards {
-			m := map[string]interface{}{
-				"Key": s.Key,
-				"State": func() string {
-					if c, ok := colors[s.State]; ok {
-						return color.New(c).Sprint(s.State)
-					}
-					return s.State
-				}(),
-				"Error": s.Error,
-			}
-			tw.Write(m)
-		}
-
-		return tw.Flush(os.Stdout)
+		return printTableShards(shards)
 	},
 }
 
@@ -265,3 +236,80 @@ var dagstoreGcCmd = &cli.Command{
 		return nil
 	},
 }
+
+func printTableShards(shards []api.DagstoreShardInfo) error {
+	if len(shards) == 0 {
+		return nil
+	}
+
+	tw := tablewriter.New(
+		tablewriter.Col("Key"),
+		tablewriter.Col("State"),
+		tablewriter.Col("Error"),
+	)
+
+	colors := map[string]color.Attribute{
+		"ShardStateAvailable": color.FgGreen,
+		"ShardStateServing":   color.FgBlue,
+		"ShardStateErrored":   color.FgRed,
+		"ShardStateNew":       color.FgYellow,
+	}
+
+	for _, s := range shards {
+		m := map[string]interface{}{
+			"Key": s.Key,
+			"State": func() string {
+				if c, ok := colors[s.State]; ok {
+					return color.New(c).Sprint(s.State)
+				}
+				return s.State
+			}(),
+			"Error": s.Error,
+		}
+		tw.Write(m)
+	}
+	return tw.Flush(os.Stdout)
+}
+
+var dagstoreLookupPiecesCmd = &cli.Command{
+	Name:      "lookup-pieces",
+	Usage:     "Lookup pieces that a given CID belongs to",
+	ArgsUsage: "<cid>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide a CID")
+		}
+
+		cidStr := cctx.Args().First()
+		cid, err := cid.Parse(cidStr)
+		if err != nil {
+			return fmt.Errorf("invalid CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shards, err := marketsApi.DagstoreLookupPieces(ctx, cid)
+		if err != nil {
+			return err
+		}
+
+		return printTableShards(shards)
+	},
+}

cmd/lotus-miner/index_provider.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/fatih/color"
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var indexProvCmd = &cli.Command{
+	Name:  "index",
+	Usage: "Manage the index provider on the markets subsystem",
+	Subcommands: []*cli.Command{
+		indexProvAnnounceCmd,
+		indexProvAnnounceAllCmd,
+	},
+}
+
+var indexProvAnnounceCmd = &cli.Command{
+	Name:      "announce",
+	ArgsUsage: "<deal proposal cid>",
+	Usage:     "Announce a deal to indexers so they can download its index",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide the deal proposal CID")
+		}
+
+		proposalCidStr := cctx.Args().First()
+		proposalCid, err := cid.Parse(proposalCidStr)
+		if err != nil {
+			return fmt.Errorf("invalid deal proposal CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return marketsApi.IndexerAnnounceDeal(ctx, proposalCid)
+	},
+}
+
+var indexProvAnnounceAllCmd = &cli.Command{
+	Name:  "announce-all",
+	Usage: "Announce all active deals to indexers so they can download its indices",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return marketsApi.IndexerAnnounceAllDeals(ctx)
+	},
+}

cmd/lotus-miner/info.go

@@ -126,7 +126,7 @@ func infoCmdAct(cctx *cli.Context) error {
 
 	alerts, err := minerApi.LogAlerts(ctx)
 	if err != nil {
-		return xerrors.Errorf("getting alerts: %w", err)
+		fmt.Printf("ERROR: getting alerts: %s\n", err)
 	}
 
 	activeAlerts := make([]alerting.Alert, 0)

cmd/lotus-miner/info_all.go

@@ -96,6 +96,11 @@ var infoAllCmd = &cli.Command{
 			fmt.Println("ERROR: ", err)
 		}
 
+		fmt.Println("\n#: Storage Locks")
+		if err := storageLocks.Action(cctx); err != nil {
+			fmt.Println("ERROR: ", err)
+		}
+
 		fmt.Println("\n#: Sched Diag")
 		if err := sealingSchedDiagCmd.Action(cctx); err != nil {
 			fmt.Println("ERROR: ", err)
@@ -192,6 +197,11 @@ var infoAllCmd = &cli.Command{
 			fmt.Println("ERROR: ", err)
 		}
 
+		fmt.Println("\n#: Storage Sector List")
+		if err := storageListSectorsCmd.Action(cctx); err != nil {
+			fmt.Println("ERROR: ", err)
+		}
+
 		fmt.Println("\n#: Expired Sectors")
 		if err := sectorsExpiredCmd.Action(cctx); err != nil {
 			fmt.Println("ERROR: ", err)

cmd/lotus-miner/init.go

@@ -467,12 +467,15 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			stor := stores.NewRemote(lstor, si, http.Header(sa), 10, &stores.DefaultPartialFileHandler{})
 
 			smgr, err := sectorstorage.New(ctx, lstor, stor, lr, si, sectorstorage.SealerConfig{
-				ParallelFetchLimit: 10,
-				AllowAddPiece:      true,
-				AllowPreCommit1:    true,
-				AllowPreCommit2:    true,
-				AllowCommit:        true,
-				AllowUnseal:        true,
+				ParallelFetchLimit:       10,
+				AllowAddPiece:            true,
+				AllowPreCommit1:          true,
+				AllowPreCommit2:          true,
+				AllowCommit:              true,
+				AllowUnseal:              true,
+				AllowReplicaUpdate:       true,
+				AllowProveReplicaUpdate2: true,
+				AllowRegenSectorKey:      true,
 			}, wsts, smsts)
 			if err != nil {
 				return err

cmd/lotus-miner/main.go

@@ -48,6 +48,7 @@ func main() {
 		lcli.WithCategory("market", retrievalDealsCmd),
 		lcli.WithCategory("market", dataTransfersCmd),
 		lcli.WithCategory("market", dagstoreCmd),
+		lcli.WithCategory("market", indexProvCmd),
 		lcli.WithCategory("storage", sectorsCmd),
 		lcli.WithCategory("storage", provingCmd),
 		lcli.WithCategory("storage", storageCmd),

cmd/lotus-miner/proving.go

@@ -437,6 +437,7 @@ var provingCheckProvableCmd = &cli.Command{
 			}
 
 			var tocheck []storage.SectorRef
+			var update []bool
 			for _, info := range sectorInfos {
 				si := abi.SectorID{
 					Miner:  abi.ActorID(mid),
@@ -454,9 +455,10 @@ var provingCheckProvableCmd = &cli.Command{
 					ProofType: info.SealProof,
 					ID:        si,
 				})
+				update = append(update, info.SectorKeyCID != nil)
 			}
 
-			bad, err := sapi.CheckProvable(ctx, info.WindowPoStProofType, tocheck, cctx.Bool("slow"))
+			bad, err := sapi.CheckProvable(ctx, info.WindowPoStProofType, tocheck, update, cctx.Bool("slow"))
 			if err != nil {
 				return err
 			}

cmd/lotus-miner/sealing.go

@@ -39,9 +39,12 @@ func barString(total, y, g float64) string {
 	yBars := int(math.Round(y / total * barCols))
 	gBars := int(math.Round(g / total * barCols))
 	eBars := int(barCols) - yBars - gBars
-	return color.YellowString(strings.Repeat("|", yBars)) +
-		color.GreenString(strings.Repeat("|", gBars)) +
-		strings.Repeat(" ", eBars)
+	var barString = color.YellowString(strings.Repeat("|", yBars)) +
+		color.GreenString(strings.Repeat("|", gBars))
+	if eBars >= 0 {
+		barString += strings.Repeat(" ", eBars)
+	}
+	return barString
 }
 
 var sealingWorkersCmd = &cli.Command{

cmd/lotus-miner/sectors.go

@@ -161,7 +161,7 @@ var sectorsStatusCmd = &cli.Command{
 			fmt.Printf("Expiration:\t\t%v\n", status.Expiration)
 			fmt.Printf("DealWeight:\t\t%v\n", status.DealWeight)
 			fmt.Printf("VerifiedDealWeight:\t\t%v\n", status.VerifiedDealWeight)
-			fmt.Printf("InitialPledge:\t\t%v\n", status.InitialPledge)
+			fmt.Printf("InitialPledge:\t\t%v\n", types.FIL(status.InitialPledge))
 			fmt.Printf("\nExpiration Info\n")
 			fmt.Printf("OnTime:\t\t%v\n", status.OnTime)
 			fmt.Printf("Early:\t\t%v\n", status.Early)
@@ -294,8 +294,14 @@ var sectorsListCmd = &cli.Command{
 			Aliases: []string{"e"},
 		},
 		&cli.BoolFlag{
-			Name:  "seal-time",
-			Usage: "display how long it took for the sector to be sealed",
+			Name:    "initial-pledge",
+			Usage:   "display initial pledge",
+			Aliases: []string{"p"},
+		},
+		&cli.BoolFlag{
+			Name:    "seal-time",
+			Usage:   "display how long it took for the sector to be sealed",
+			Aliases: []string{"t"},
 		},
 		&cli.StringFlag{
 			Name:  "states",
@@ -405,6 +411,7 @@ var sectorsListCmd = &cli.Command{
 			tablewriter.Col("Deals"),
 			tablewriter.Col("DealWeight"),
 			tablewriter.Col("VerifiedPower"),
+			tablewriter.Col("Pledge"),
 			tablewriter.NewLineCol("Error"),
 			tablewriter.NewLineCol("RecoveryTimeout"))
 
@@ -483,6 +490,9 @@ var sectorsListCmd = &cli.Command{
 						m["RecoveryTimeout"] = color.YellowString(lcli.EpochTime(head.Height(), st.Early))
 					}
 				}
+				if inSSet && cctx.Bool("initial-pledge") {
+					m["Pledge"] = types.FIL(st.InitialPledge).Short()
+				}
 			}
 
 			if !fast && deals > 0 {
@@ -1525,11 +1535,23 @@ var sectorsSnapAbortCmd = &cli.Command{
 	Name:      "abort-upgrade",
 	Usage:     "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before",
 	ArgsUsage: "<sectorNum>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "pass this flag if you know what you are doing",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
 			return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))
 		}
 
+		really := cctx.Bool("really-do-it")
+		if !really {
+			//nolint:golint
+			return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned")
+		}
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err

cmd/lotus-miner/storage.go

@@ -368,6 +368,7 @@ type storedSector struct {
 	store stores.SectorStorageInfo
 
 	unsealed, sealed, cache bool
+	update, updatecache     bool
 }
 
 var storageFindCmd = &cli.Command{
@@ -421,6 +422,16 @@ var storageFindCmd = &cli.Command{
 			return xerrors.Errorf("finding cache: %w", err)
 		}
 
+		us, err := nodeApi.StorageFindSector(ctx, sid, storiface.FTUpdate, 0, false)
+		if err != nil {
+			return xerrors.Errorf("finding sealed: %w", err)
+		}
+
+		uc, err := nodeApi.StorageFindSector(ctx, sid, storiface.FTUpdateCache, 0, false)
+		if err != nil {
+			return xerrors.Errorf("finding cache: %w", err)
+		}
+
 		byId := map[stores.ID]*storedSector{}
 		for _, info := range u {
 			sts, ok := byId[info.ID]
@@ -455,6 +466,28 @@ var storageFindCmd = &cli.Command{
 			}
 			sts.cache = true
 		}
+		for _, info := range us {
+			sts, ok := byId[info.ID]
+			if !ok {
+				sts = &storedSector{
+					id:    info.ID,
+					store: info,
+				}
+				byId[info.ID] = sts
+			}
+			sts.update = true
+		}
+		for _, info := range uc {
+			sts, ok := byId[info.ID]
+			if !ok {
+				sts = &storedSector{
+					id:    info.ID,
+					store: info,
+				}
+				byId[info.ID] = sts
+			}
+			sts.updatecache = true
+		}
 
 		local, err := nodeApi.StorageLocal(ctx)
 		if err != nil {
@@ -480,6 +513,12 @@ var storageFindCmd = &cli.Command{
 			if info.cache {
 				types += "Cache, "
 			}
+			if info.update {
+				types += "Update, "
+			}
+			if info.updatecache {
+				types += "UpdateCache, "
+			}
 
 			fmt.Printf("In %s (%s)\n", info.id, types[:len(types)-2])
 			fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanStore)

cmd/lotus-seal-worker/main.go

@@ -173,6 +173,11 @@ var runCmd = &cli.Command{
 			Usage: "enable prove replica update 2",
 			Value: true,
 		},
+		&cli.BoolFlag{
+			Name:  "regen-sector-key",
+			Usage: "enable regen sector key",
+			Value: true,
+		},
 		&cli.IntFlag{
 			Name:  "parallel-fetch-limit",
 			Usage: "maximum fetch operations to run in parallel",
@@ -278,12 +283,15 @@ var runCmd = &cli.Command{
 		if cctx.Bool("commit") {
 			taskTypes = append(taskTypes, sealtasks.TTCommit2)
 		}
-		if cctx.Bool("replicaupdate") {
+		if cctx.Bool("replica-update") {
 			taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate)
 		}
 		if cctx.Bool("prove-replica-update2") {
 			taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2)
 		}
+		if cctx.Bool("regen-sector-key") {
+			taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey)
+		}
 
 		if len(taskTypes) == 0 {
 			return xerrors.Errorf("no task types specified")

cmd/lotus-seal-worker/tasks.go

@@ -22,11 +22,14 @@ var tasksCmd = &cli.Command{
 }
 
 var allowSetting = map[sealtasks.TaskType]struct{}{
-	sealtasks.TTAddPiece:   {},
-	sealtasks.TTPreCommit1: {},
-	sealtasks.TTPreCommit2: {},
-	sealtasks.TTCommit2:    {},
-	sealtasks.TTUnseal:     {},
+	sealtasks.TTAddPiece:            {},
+	sealtasks.TTPreCommit1:          {},
+	sealtasks.TTPreCommit2:          {},
+	sealtasks.TTCommit2:             {},
+	sealtasks.TTUnseal:              {},
+	sealtasks.TTReplicaUpdate:       {},
+	sealtasks.TTProveReplicaUpdate2: {},
+	sealtasks.TTRegenSectorKey:      {},
 }
 
 var settableStr = func() string {

cmd/lotus-seed/genesis.go

@@ -508,12 +508,19 @@ var genesisSetRemainderCmd = &cli.Command{
 }
 
 var genesisSetActorVersionCmd = &cli.Command{
-	Name:      "set-network-version",
-	Usage:     "Set the version that this network will start from",
-	ArgsUsage: "<genesisFile> <actorVersion>",
+	Name:  "set-network-version",
+	Usage: "Set the version that this network will start from",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "network-version",
+			Usage: "network version to start genesis with",
+			Value: int(build.GenesisNetworkVersion),
+		},
+	},
+	ArgsUsage: "<genesisFile>",
 	Action: func(cctx *cli.Context) error {
-		if cctx.Args().Len() != 2 {
-			return fmt.Errorf("must specify genesis file and network version (e.g. '0'")
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify genesis file")
 		}
 
 		genf, err := homedir.Expand(cctx.Args().First())
@@ -531,16 +538,12 @@ var genesisSetActorVersionCmd = &cli.Command{
 			return xerrors.Errorf("unmarshal genesis template: %w", err)
 		}
 
-		nv, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
-		if err != nil {
-			return xerrors.Errorf("parsing network version: %w", err)
-		}
-
-		if nv > uint64(build.NewestNetworkVersion) {
+		nv := network.Version(cctx.Int("network-version"))
+		if nv > build.NewestNetworkVersion {
 			return xerrors.Errorf("invalid network version: %d", nv)
 		}
 
-		template.NetworkVersion = network.Version(nv)
+		template.NetworkVersion = nv
 
 		b, err = json.MarshalIndent(&template, "", "  ")
 		if err != nil {

cmd/lotus-shed/diff.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var diffCmd = &cli.Command{
+	Name:        "diff",
+	Usage:       "diff state objects",
+	Subcommands: []*cli.Command{diffStateTrees},
+}
+
+var diffStateTrees = &cli.Command{
+	Name:      "state-trees",
+	Usage:     "diff two state-trees",
+	ArgsUsage: "<state-tree-a> <state-tree-b>",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if cctx.NArg() != 2 {
+			return xerrors.Errorf("expected two state-tree roots")
+		}
+
+		argA := cctx.Args().Get(1)
+		rootA, err := cid.Parse(argA)
+		if err != nil {
+			return xerrors.Errorf("first state-tree root (%q) is not a CID: %w", argA, err)
+		}
+		argB := cctx.Args().Get(1)
+		rootB, err := cid.Parse(argB)
+		if err != nil {
+			return xerrors.Errorf("second state-tree root (%q) is not a CID: %w", argB, err)
+		}
+
+		if rootA == rootB {
+			fmt.Println("state trees do not differ")
+			return nil
+		}
+
+		changedB, err := api.StateChangedActors(ctx, rootA, rootB)
+		if err != nil {
+			return err
+		}
+		changedA, err := api.StateChangedActors(ctx, rootB, rootA)
+		if err != nil {
+			return err
+		}
+
+		diff := func(stateA, stateB types.Actor) {
+			if stateB.Code != stateA.Code {
+				fmt.Printf("  code: %s != %s\n", stateA.Code, stateB.Code)
+			}
+			if stateB.Head != stateA.Head {
+				fmt.Printf("  state: %s != %s\n", stateA.Head, stateB.Head)
+			}
+			if stateB.Nonce != stateA.Nonce {
+				fmt.Printf("  nonce: %d != %d\n", stateA.Nonce, stateB.Nonce)
+			}
+			if !stateB.Balance.Equals(stateA.Balance) {
+				fmt.Printf("  balance: %s != %s\n", stateA.Balance, stateB.Balance)
+			}
+		}
+
+		fmt.Printf("state differences between %s (first) and %s (second):\n\n", rootA, rootB)
+		for addr, stateA := range changedA {
+			fmt.Println(addr)
+			stateB, ok := changedB[addr]
+			if ok {
+				diff(stateA, stateB)
+				continue
+			} else {
+				fmt.Printf("  actor does not exist in second state-tree (%s)\n", rootB)
+			}
+			fmt.Println()
+			delete(changedB, addr)
+		}
+		for addr, stateB := range changedB {
+			fmt.Println(addr)
+			stateA, ok := changedA[addr]
+			if ok {
+				diff(stateA, stateB)
+				continue
+			} else {
+				fmt.Printf("  actor does not exist in first state-tree (%s)\n", rootA)
+			}
+			fmt.Println()
+		}
+		return nil
+	},
+}

cmd/lotus-shed/main.go

@@ -68,6 +68,7 @@ func main() {
 		sendCsvCmd,
 		terminationsCmd,
 		migrationsCmd,
+		diffCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/verifreg.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
+
+	"github.com/filecoin-project/go-state-types/crypto"
+
 	"github.com/filecoin-project/go-state-types/big"
 
 	"github.com/urfave/cli/v2"
@@ -35,6 +40,7 @@ var verifRegCmd = &cli.Command{
 		verifRegListClientsCmd,
 		verifRegCheckClientCmd,
 		verifRegCheckVerifierCmd,
+		verifRegRemoveVerifiedClientDataCapCmd,
 	},
 }
 
@@ -409,3 +415,154 @@ var verifRegCheckVerifierCmd = &cli.Command{
 		return nil
 	},
 }
+
+var verifRegRemoveVerifiedClientDataCapCmd = &cli.Command{
+	Name:      "remove-verified-client-data-cap",
+	Usage:     "Remove data cap from verified client",
+	ArgsUsage: "<message sender> <client address> <allowance to remove> <verifier 1 address> <verifier 1 signature> <verifier 2 address> <verifier 2 signature>",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 7 {
+			return fmt.Errorf("must specify seven arguments: sender, client, allowance to remove, verifier 1 address, verifier 1 signature, verifier 2 address, verifier 2 signature")
+		}
+
+		srv, err := lcli.GetFullNodeServices(cctx)
+		if err != nil {
+			return err
+		}
+		defer srv.Close() //nolint:errcheck
+
+		api := srv.FullNodeAPI()
+		ctx := lcli.ReqContext(cctx)
+
+		sender, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		client, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		allowanceToRemove, err := types.BigFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		verifier1Addr, err := address.NewFromString(cctx.Args().Get(3))
+		if err != nil {
+			return err
+		}
+
+		verifier1Sig, err := hex.DecodeString(cctx.Args().Get(4))
+		if err != nil {
+			return err
+		}
+
+		verifier2Addr, err := address.NewFromString(cctx.Args().Get(5))
+		if err != nil {
+			return err
+		}
+
+		verifier2Sig, err := hex.DecodeString(cctx.Args().Get(6))
+		if err != nil {
+			return err
+		}
+
+		var sig1 crypto.Signature
+		if err := sig1.UnmarshalBinary(verifier1Sig); err != nil {
+			return xerrors.Errorf("couldn't unmarshal sig: %w", err)
+		}
+
+		var sig2 crypto.Signature
+		if err := sig2.UnmarshalBinary(verifier2Sig); err != nil {
+			return xerrors.Errorf("couldn't unmarshal sig: %w", err)
+		}
+
+		params, err := actors.SerializeParams(&verifreg.RemoveDataCapParams{
+			VerifiedClientToRemove: client,
+			DataCapAmountToRemove:  allowanceToRemove,
+			VerifierRequest1: verifreg.RemoveDataCapRequest{
+				Verifier:          verifier1Addr,
+				VerifierSignature: sig1,
+			},
+			VerifierRequest2: verifreg.RemoveDataCapRequest{
+				Verifier:          verifier2Addr,
+				VerifierSignature: sig2,
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		vrk, err := api.StateVerifiedRegistryRootKey(ctx, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		vrkState, err := api.StateGetActor(ctx, vrk, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		apibs := blockstore.NewAPIBlockstore(api)
+		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
+
+		st, err := multisig.Load(store, vrkState)
+		if err != nil {
+			return err
+		}
+
+		signers, err := st.Signers()
+		if err != nil {
+			return err
+		}
+
+		senderIsSigner := false
+		senderIdAddr, err := address.IDFromAddress(sender)
+		if err != nil {
+			return err
+		}
+
+		for _, signer := range signers {
+			signerIdAddr, err := address.IDFromAddress(signer)
+			if err != nil {
+				return err
+			}
+
+			if signerIdAddr == senderIdAddr {
+				senderIsSigner = true
+			}
+		}
+
+		if !senderIsSigner {
+			return fmt.Errorf("sender must be a vrk signer")
+		}
+
+		proto, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(verifreg.Methods.RemoveVerifiedClientDataCap), params)
+		if err != nil {
+			return err
+		}
+
+		sm, _, err := srv.PublishMessage(ctx, proto, false)
+		if err != nil {
+			return err
+		}
+
+		msgCid := sm.Cid()
+
+		fmt.Printf("message sent, now waiting on cid: %s\n", msgCid)
+
+		mwait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
+		if err != nil {
+			return err
+		}
+
+		if mwait.Receipt.ExitCode != 0 {
+			return fmt.Errorf("failed to removed verified data cap: %d", mwait.Receipt.ExitCode)
+		}
+
+		//TODO: Internal msg might still have failed
+		return nil
+	},
+}

documentation/en/api-v0-methods-miner.md

@@ -23,6 +23,7 @@
   * [DagstoreInitializeAll](#DagstoreInitializeAll)
   * [DagstoreInitializeShard](#DagstoreInitializeShard)
   * [DagstoreListShards](#DagstoreListShards)
+  * [DagstoreLookupPieces](#DagstoreLookupPieces)
   * [DagstoreRecoverShard](#DagstoreRecoverShard)
 * [Deals](#Deals)
   * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
@@ -43,6 +44,9 @@
   * [DealsSetPieceCidBlocklist](#DealsSetPieceCidBlocklist)
 * [I](#I)
   * [ID](#ID)
+* [Indexer](#Indexer)
+  * [IndexerAnnounceAllDeals](#IndexerAnnounceAllDeals)
+  * [IndexerAnnounceDeal](#IndexerAnnounceDeal)
 * [Log](#Log)
   * [LogAlerts](#LogAlerts)
   * [LogList](#LogList)
@@ -84,6 +88,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -338,6 +345,9 @@ Inputs:
       "ProofType": 8
     }
   ],
+  [
+    true
+  ],
   true
 ]
 ```
@@ -514,6 +524,32 @@ Response:
 ]
 ```
 
+### DagstoreLookupPieces
+DagstoreLookupPieces returns information about shards that contain the given CID.
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response:
+```json
+[
+  {
+    "Key": "baga6ea4seaqecmtz7iak33dsfshi627abz4i4665dfuzr3qfs4bmad6dx3iigdq",
+    "State": "ShardStateAvailable",
+    "Error": "\u003cerror\u003e"
+  }
+]
+```
+
 ### DagstoreRecoverShard
 DagstoreRecoverShard attempts to recover a failed shard.
 
@@ -772,6 +808,37 @@ Inputs: `null`
 
 Response: `"12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"`
 
+## Indexer
+
+
+### IndexerAnnounceAllDeals
+IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `{}`
+
+### IndexerAnnounceDeal
+IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+so they can download its index
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `{}`
+
 ## Log
 
 
@@ -1784,6 +1851,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 

documentation/en/api-v0-methods.md

@@ -131,6 +131,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -3905,6 +3908,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 

documentation/en/api-v1-unstable-methods.md

@@ -137,6 +137,9 @@
   * [NetLimit](#NetLimit)
   * [NetPeerInfo](#NetPeerInfo)
   * [NetPeers](#NetPeers)
+  * [NetProtectAdd](#NetProtectAdd)
+  * [NetProtectList](#NetProtectList)
+  * [NetProtectRemove](#NetProtectRemove)
   * [NetPubsubScores](#NetPubsubScores)
   * [NetSetLimit](#NetSetLimit)
   * [NetStat](#NetStat)
@@ -4266,6 +4269,52 @@ Response:
 ]
 ```
 
+### NetProtectAdd
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
+### NetProtectList
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+[
+  "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+]
+```
+
+### NetProtectRemove
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "12D3KooWGzxzKZYveHXtpG6AsrUJBcWxHBFS2HsEoGTxrMLvKXtf"
+  ]
+]
+```
+
+Response: `{}`
+
 ### NetPubsubScores
 
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-dev
+   1.15.1-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -30,6 +30,7 @@ COMMANDS:
      retrieval-deals  Manage retrieval deals and related configuration
      data-transfers   Manage data transfers
      dagstore         Manage the dagstore on the markets subsystem
+     index            Manage the index provider on the markets subsystem
    NETWORK:
      net  Manage P2P Network
    RETRIEVAL:
@@ -78,7 +79,6 @@ OPTIONS:
    --gas-premium value          set gas premium for initialization messages in AttoFIL (default: "0")
    --from value                 select which address to send actor creation message from
    --help, -h                   show help (default: false)
-   --version, -v                print the version (default: false)
    
 ```
 
@@ -160,8 +160,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -203,11 +202,11 @@ USAGE:
 
 DESCRIPTION:
    The backup command writes a copy of node metadata under the specified path
-
-Online backups:
-For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
-to a path where backup files are supposed to be saved, and the path specified in
-this command must be within this base path
+   
+   Online backups:
+   For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
+   to a path where backup files are supposed to be saved, and the path specified in
+   this command must be within this base path
 
 OPTIONS:
    --offline   create backup without the node running (default: false)
@@ -249,8 +248,7 @@ COMMANDS:
    help, h                Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -339,8 +337,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -433,7 +430,6 @@ OPTIONS:
    --hide-sectors-info  hide sectors info (default: false)
    --blocks value       Log of produced <blocks> newest blocks and rewards(Miner Fee excluded) (default: 0)
    --help, -h           show help (default: false)
-   --version, -v        print the version (default: false)
    
 ```
 
@@ -464,8 +460,7 @@ COMMANDS:
    help, h       Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -512,8 +507,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -540,23 +534,22 @@ USAGE:
 
 DESCRIPTION:
    Set the log level for logging systems:
-
-   The system flag can be specified multiple times.
-
-   eg) log set-level --system chain --system chainxchg debug
-
-   Available Levels:
-   debug
-   info
-   warn
-   error
-
-   Environment Variables:
-   GOLOG_LOG_LEVEL - Default log level for all log systems
-   GOLOG_LOG_FMT   - Change output log format (json, nocolor)
-   GOLOG_FILE      - Write logs to file
-   GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
-
+   
+      The system flag can be specified multiple times.
+   
+      eg) log set-level --system chain --system chainxchg debug
+   
+      Available Levels:
+      debug
+      info
+      warn
+      error
+   
+      Environment Variables:
+      GOLOG_LOG_LEVEL - Default log level for all log systems
+      GOLOG_LOG_FMT   - Change output log format (json, nocolor)
+      GOLOG_FILE      - Write logs to file
+      GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
 
 OPTIONS:
    --system value  limit to log system
@@ -634,8 +627,7 @@ COMMANDS:
    help, h            Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -683,8 +675,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -856,8 +847,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -876,8 +866,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -981,8 +970,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1062,11 +1050,11 @@ COMMANDS:
    recover-shard     Attempt to recover a shard in errored state
    initialize-all    Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
    gc                Garbage collect the dagstore
+   lookup-pieces     Lookup pieces that a given CID belongs to
    help, h           Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1142,6 +1130,66 @@ OPTIONS:
    
 ```
 
+### lotus-miner dagstore lookup-pieces
+```
+NAME:
+   lotus-miner dagstore lookup-pieces - Lookup pieces that a given CID belongs to
+
+USAGE:
+   lotus-miner dagstore lookup-pieces [command options] <cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+## lotus-miner index
+```
+NAME:
+   lotus-miner index - Manage the index provider on the markets subsystem
+
+USAGE:
+   lotus-miner index command [command options] [arguments...]
+
+COMMANDS:
+   announce      Announce a deal to indexers so they can download its index
+   announce-all  Announce all active deals to indexers so they can download its indices
+   help, h       Shows a list of commands or help for one command
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner index announce
+```
+NAME:
+   lotus-miner index announce - Announce a deal to indexers so they can download its index
+
+USAGE:
+   lotus-miner index announce [command options] <deal proposal cid>
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner index announce-all
+```
+NAME:
+   lotus-miner index announce-all - Announce all active deals to indexers so they can download its indices
+
+USAGE:
+   lotus-miner index announce-all [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner net
 ```
 NAME:
@@ -1151,22 +1199,24 @@ USAGE:
    lotus-miner net command [command options] [arguments...]
 
 COMMANDS:
-   peers         Print peers
-   connect       Connect to a peer
-   listen        List listen addresses
-   id            Get node identity
-   findpeer      Find the addresses of a given peerID
-   scores        Print peers' pubsub scores
-   reachability  Print information about reachability from the internet
-   bandwidth     Print bandwidth usage information
-   block         Manage network connection gating rules
-   stat          Report resource usage for a scope
-   limit         Get or set resource limits for a scope
-   help, h       Shows a list of commands or help for one command
+   peers           Print peers
+   connect         Connect to a peer
+   listen          List listen addresses
+   id              Get node identity
+   findpeer        Find the addresses of a given peerID
+   scores          Print peers' pubsub scores
+   reachability    Print information about reachability from the internet
+   bandwidth       Print bandwidth usage information
+   block           Manage network connection gating rules
+   stat            Report resource usage for a scope
+   limit           Get or set resource limits for a scope
+   protect         Add one or more peer IDs to the list of protected peer connections
+   unprotect       Remove one or more peer IDs from the list of protected peer connections.
+   list-protected  List the peer IDs with protected connection.
+   help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1294,8 +1344,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1314,8 +1363,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1373,8 +1421,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1440,15 +1487,14 @@ USAGE:
 
 DESCRIPTION:
    Report resource usage for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-  - all           -- reports the resource usage for all currently active scopes.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+     - all           -- reports the resource usage for all currently active scopes.
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -1465,16 +1511,15 @@ USAGE:
 
 DESCRIPTION:
    Get or set resource limits for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-
- The limit is json-formatted, with the same structure as the limits file.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+   
+    The limit is json-formatted, with the same structure as the limits file.
 
 OPTIONS:
    --set       set the limit for a scope (default: false)
@@ -1482,6 +1527,45 @@ OPTIONS:
    
 ```
 
+### lotus-miner net protect
+```
+NAME:
+   lotus-miner net protect - Add one or more peer IDs to the list of protected peer connections
+
+USAGE:
+   lotus-miner net protect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner net unprotect
+```
+NAME:
+   lotus-miner net unprotect - Remove one or more peer IDs from the list of protected peer connections.
+
+USAGE:
+   lotus-miner net unprotect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus-miner net list-protected
+```
+NAME:
+   lotus-miner net list-protected - List the peer IDs with protected connection.
+
+USAGE:
+   lotus-miner net list-protected [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus-miner pieces
 ```
 NAME:
@@ -1501,8 +1585,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1590,8 +1673,7 @@ COMMANDS:
    help, h               Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1621,14 +1703,15 @@ USAGE:
    lotus-miner sectors list [command options] [arguments...]
 
 OPTIONS:
-   --show-removed, -r  show removed sectors (default: false)
-   --color, -c         use color in display output (default: depends on output being a TTY)
-   --fast, -f          don't show on-chain info for better performance (default: false)
-   --events, -e        display number of events the sector has received (default: false)
-   --seal-time         display how long it took for the sector to be sealed (default: false)
-   --states value      filter sectors by a comma-separated list of states
-   --unproven, -u      only show sectors which aren't in the 'Proving' state (default: false)
-   --help, -h          show help (default: false)
+   --show-removed, -r    show removed sectors (default: false)
+   --color, -c           use color in display output (default: depends on output being a TTY)
+   --fast, -f            don't show on-chain info for better performance (default: false)
+   --events, -e          display number of events the sector has received (default: false)
+   --initial-pledge, -p  display initial pledge (default: false)
+   --seal-time, -t       display how long it took for the sector to be sealed (default: false)
+   --states value        filter sectors by a comma-separated list of states
+   --unproven, -u        only show sectors which aren't in the 'Proving' state (default: false)
+   --help, -h            show help (default: false)
    
 ```
 
@@ -1759,7 +1842,6 @@ COMMANDS:
 OPTIONS:
    --really-do-it  pass this flag if you know what you are doing (default: false)
    --help, -h      show help (default: false)
-   --version, -v   print the version (default: false)
    
 ```
 
@@ -1825,7 +1907,8 @@ USAGE:
    lotus-miner sectors abort-upgrade [command options] <sectorNum>
 
 OPTIONS:
-   --help, -h  show help (default: false)
+   --really-do-it  pass this flag if you know what you are doing (default: false)
+   --help, -h      show help (default: false)
    
 ```
 
@@ -1896,8 +1979,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1959,8 +2041,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2042,9 +2123,9 @@ USAGE:
 
 DESCRIPTION:
    Sectors can be stored across many filesystem paths. These
-commands provide ways to manage the storage the miner will used to store sectors
-long term for proving (references as 'store') as well as how sectors will be
-stored while moving through the sealing pipeline (references as 'seal').
+   commands provide ways to manage the storage the miner will used to store sectors
+   long term for proving (references as 'store') as well as how sectors will be
+   stored while moving through the sealing pipeline (references as 'seal').
 
 COMMANDS:
    attach   attach local storage path
@@ -2055,8 +2136,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2070,24 +2150,23 @@ USAGE:
 
 DESCRIPTION:
    Storage can be attached to the miner using this command. The storage volume
-list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not
-recommend manually modifying this value without further understanding of the
-storage system.
-
-Each storage volume contains a configuration file which describes the
-capabilities of the volume. When the '--init' flag is provided, this file will
-be created using the additional flags.
-
-Weight
-A high weight value means data will be more likely to be stored in this path
-
-Seal
-Data for the sealing process will be stored here
-
-Store
-Finalized sectors that will be moved here for long term storage and be proven
-over time
+   list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not
+   recommend manually modifying this value without further understanding of the
+   storage system.
    
+   Each storage volume contains a configuration file which describes the
+   capabilities of the volume. When the '--init' flag is provided, this file will
+   be created using the additional flags.
+   
+   Weight
+   A high weight value means data will be more likely to be stored in this path
+   
+   Seal
+   Data for the sealing process will be stored here
+   
+   Store
+   Finalized sectors that will be moved here for long term storage and be proven
+   over time
 
 OPTIONS:
    --init               initialize the path first (default: false)
@@ -2114,9 +2193,8 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --color        use color in display output (default: depends on output being a TTY)
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2190,8 +2268,7 @@ COMMANDS:
    help, h     Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-dev
+   1.15.1-dev
 
 COMMANDS:
    run         Start lotus worker
@@ -46,6 +46,7 @@ OPTIONS:
    --commit                      enable commit (32G sectors: all cores or GPUs, 128GiB Memory + 64GiB swap) (default: true)
    --replica-update              enable replica update (default: true)
    --prove-replica-update2       enable prove replica update 2 (default: true)
+   --regen-sector-key            enable regen sector key (default: true)
    --parallel-fetch-limit value  maximum fetch operations to run in parallel (default: 5)
    --timeout value               used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function (default: "30m")
    --help, -h                    show help (default: false)
@@ -78,8 +79,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -159,8 +159,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -170,7 +169,7 @@ NAME:
    lotus-worker tasks enable - Enable a task type
 
 USAGE:
-   lotus-worker tasks enable [command options] [UNS|C2|PC2|PC1|AP]
+   lotus-worker tasks enable [command options] [UNS|C2|PC2|PC1|PR2|RU|AP|GSK]
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -183,7 +182,7 @@ NAME:
    lotus-worker tasks disable - Disable a task type
 
 USAGE:
-   lotus-worker tasks disable [command options] [UNS|C2|PC2|PC1|AP]
+   lotus-worker tasks disable [command options] [UNS|C2|PC2|PC1|PR2|RU|AP|GSK]
 
 OPTIONS:
    --help, -h  show help (default: false)

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.0-dev
+   1.15.1-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process
@@ -71,7 +71,6 @@ OPTIONS:
    --restore value           restore from backup file
    --restore-config value    config file to use when restoring from backup
    --help, -h                show help (default: false)
-   --version, -v             print the version (default: false)
    
 ```
 
@@ -98,11 +97,11 @@ USAGE:
 
 DESCRIPTION:
    The backup command writes a copy of node metadata under the specified path
-
-Online backups:
-For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
-to a path where backup files are supposed to be saved, and the path specified in
-this command must be within this base path
+   
+   Online backups:
+   For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set
+   to a path where backup files are supposed to be saved, and the path specified in
+   this command must be within this base path
 
 OPTIONS:
    --offline   create backup without the node running (default: false)
@@ -124,8 +123,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -218,8 +216,7 @@ COMMANDS:
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -372,8 +369,7 @@ COMMANDS:
    help, h   Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -448,8 +444,7 @@ COMMANDS:
      cancel-transfer   Force cancel a data transfer
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -566,37 +561,36 @@ CATEGORY:
 
 DESCRIPTION:
    Retrieve data from the Filecoin network.
-
-The retrieve command will attempt to find a provider make a retrieval deal with
-them. In case a provider can't be found, it can be specified with the --provider
-flag.
-
-By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
-a CAR file containing the raw IPLD graph can be exported by setting the --car
-flag.
-
-Partial Retrieval:
-
-The --data-selector flag can be used to specify a sub-graph to fetch. The
-selector can be specified as either IPLD datamodel text-path selector, or IPLD
-json selector.
-
-In case of unixfs retrieval, the selector must point at a single root node, and
-match the entire graph under that node.
-
-In case of CAR retrieval, the selector must have one common "sub-root" node.
-
-Examples:
-
-- Retrieve a file by CID
-  $ lotus client retrieve Qm... my-file.txt
-
-- Retrieve a file by CID from f0123
-  $ lotus client retrieve --provider f0123 Qm... my-file.txt
-
-- Retrieve a first file from a specified directory
-  $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
-
+   
+   The retrieve command will attempt to find a provider make a retrieval deal with
+   them. In case a provider can't be found, it can be specified with the --provider
+   flag.
+   
+   By default the data will be interpreted as DAG-PB UnixFSv1 File. Alternatively
+   a CAR file containing the raw IPLD graph can be exported by setting the --car
+   flag.
+   
+   Partial Retrieval:
+   
+   The --data-selector flag can be used to specify a sub-graph to fetch. The
+   selector can be specified as either IPLD datamodel text-path selector, or IPLD
+   json selector.
+   
+   In case of unixfs retrieval, the selector must point at a single root node, and
+   match the entire graph under that node.
+   
+   In case of CAR retrieval, the selector must have one common "sub-root" node.
+   
+   Examples:
+   
+   - Retrieve a file by CID
+     $ lotus client retrieve Qm... my-file.txt
+   
+   - Retrieve a file by CID from f0123
+     $ lotus client retrieve --provider f0123 Qm... my-file.txt
+   
+   - Retrieve a first file from a specified directory
+     $ lotus client retrieve --data-selector /Links/0/Hash Qm... my-file.txt
 
 OPTIONS:
    --car                                                   Export to a car file instead of a regular file (default: false)
@@ -709,13 +703,13 @@ CATEGORY:
 
 DESCRIPTION:
    Make a deal with a miner.
-dataCid comes from running 'lotus client import'.
-miner is the address of the miner you wish to make a deal with.
-price is measured in FIL/Epoch. Miners usually don't accept a bid
-lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
-with 'lotus client query-ask <miner address>'.
-duration is how long the miner should store the data for, in blocks.
-The minimum value is 518400 (6 months).
+   dataCid comes from running 'lotus client import'.
+   miner is the address of the miner you wish to make a deal with.
+   price is measured in FIL/Epoch. Miners usually don't accept a bid
+   lower than their advertised ask (which is in FIL/GiB/Epoch). You can check a miners listed price
+   with 'lotus client query-ask <miner address>'.
+   duration is how long the miner should store the data for, in blocks.
+   The minimum value is 518400 (6 months).
 
 OPTIONS:
    --manual-piece-cid value     manually specify piece commitment for data (dataCid must be to a car file)
@@ -976,7 +970,6 @@ COMMANDS:
 OPTIONS:
    --confidence value  number of block confirmations to wait for (default: 5)
    --help, -h          show help (default: false)
-   --version, -v       print the version (default: false)
    
 ```
 
@@ -1234,16 +1227,16 @@ USAGE:
    lotus filplus command [command options] [arguments...]
 
 COMMANDS:
-   grant-datacap         give allowance to the specified verified client address
-   list-notaries         list all notaries
-   list-clients          list all verified clients
-   check-client-datacap  check verified client remaining bytes
-   check-notary-datacap  check a notary's remaining bytes
-   help, h               Shows a list of commands or help for one command
+   grant-datacap                  give allowance to the specified verified client address
+   list-notaries                  list all notaries
+   list-clients                   list all verified clients
+   check-client-datacap           check verified client remaining bytes
+   check-notary-datacap           check a notary's remaining bytes
+   sign-remove-data-cap-proposal  allows a notary to sign a Remove Data Cap Proposal
+   help, h                        Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1313,6 +1306,20 @@ OPTIONS:
    
 ```
 
+### lotus filplus sign-remove-data-cap-proposal
+```
+NAME:
+   lotus filplus sign-remove-data-cap-proposal - allows a notary to sign a Remove Data Cap Proposal
+
+USAGE:
+   lotus filplus sign-remove-data-cap-proposal [command options] [arguments...]
+
+OPTIONS:
+   --id value  specify the RemoveDataCapProposal ID (will look up on chain if unspecified) (default: 0)
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus paych
 ```
 NAME:
@@ -1332,8 +1339,7 @@ COMMANDS:
    help, h            Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1382,8 +1388,7 @@ COMMANDS:
    help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1534,8 +1539,7 @@ COMMANDS:
    help, h       Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1587,8 +1591,7 @@ COMMANDS:
    help, h   Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -1643,7 +1646,7 @@ NAME:
    lotus mpool replace - replace a message in the mempool
 
 USAGE:
-   lotus mpool replace [command options] <from nonce> | <message-cid>
+   lotus mpool replace [command options] <from> <nonce> | <message-cid>
 
 OPTIONS:
    --gas-feecap value   gas feecap for new message (burn and pay to miner, attoFIL/GasUnit)
@@ -1748,7 +1751,6 @@ COMMANDS:
 OPTIONS:
    --tipset value  specify tipset to call method on (pass comma separated array of cids)
    --help, -h      show help (default: false)
-   --version, -v   print the version (default: false)
    
 ```
 
@@ -2033,8 +2035,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2119,8 +2120,7 @@ COMMANDS:
    help, h          Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2191,10 +2191,9 @@ USAGE:
 
 DESCRIPTION:
    Collect object size and ipld link count for an object.
-
-   When a base is provided it will be walked first, and all links visisted
-   will be ignored when the passed in object is walked.
-
+   
+      When a base is provided it will be walked first, and all links visisted
+      will be ignored when the passed in object is walked.
 
 OPTIONS:
    --base value  ignore links found in this obj
@@ -2244,34 +2243,33 @@ USAGE:
 
 DESCRIPTION:
    Get ipld node under a specified path:
-
-   lotus chain get /ipfs/[cid]/some/path
-
-   Path prefixes:
-   - /ipfs/[cid], /ipld/[cid] - traverse IPLD path
-   - /pstate - traverse from head.ParentStateRoot
-
-   Note:
-   You can use special path elements to traverse through some data structures:
-   - /ipfs/[cid]/@H:elem - get 'elem' from hamt
-   - /ipfs/[cid]/@Hi:123 - get varint elem 123 from hamt
-   - /ipfs/[cid]/@Hu:123 - get uvarint elem 123 from hamt
-   - /ipfs/[cid]/@Ha:t01 - get element under Addr(t01).Bytes
-   - /ipfs/[cid]/@A:10   - get 10th amt element
-   - .../@Ha:t01/@state  - get pretty map-based actor state
-
-   List of --as-type types:
-   - raw
-   - block
-   - message
-   - smessage, signedmessage
-   - actor
-   - amt
-   - hamt-epoch
-   - hamt-address
-   - cronevent
-   - account-state
-
+   
+      lotus chain get /ipfs/[cid]/some/path
+   
+      Path prefixes:
+      - /ipfs/[cid], /ipld/[cid] - traverse IPLD path
+      - /pstate - traverse from head.ParentStateRoot
+   
+      Note:
+      You can use special path elements to traverse through some data structures:
+      - /ipfs/[cid]/@H:elem - get 'elem' from hamt
+      - /ipfs/[cid]/@Hi:123 - get varint elem 123 from hamt
+      - /ipfs/[cid]/@Hu:123 - get uvarint elem 123 from hamt
+      - /ipfs/[cid]/@Ha:t01 - get element under Addr(t01).Bytes
+      - /ipfs/[cid]/@A:10   - get 10th amt element
+      - .../@Ha:t01/@state  - get pretty map-based actor state
+   
+      List of --as-type types:
+      - raw
+      - block
+      - message
+      - smessage, signedmessage
+      - actor
+      - amt
+      - hamt-epoch
+      - hamt-address
+      - cronevent
+      - account-state
 
 OPTIONS:
    --as-type value  specify type to interpret output as
@@ -2291,18 +2289,17 @@ USAGE:
 
 DESCRIPTION:
    Bisect the chain state tree:
-
-   lotus chain bisect [min height] [max height] '1/2/3/state/path' 'shell command' 'args'
-
-   Returns the first tipset in which condition is true
-                  v
-   [start] FFFFFFFTTT [end]
-
-   Example: find height at which deal ID 100 000 appeared
-    - lotus chain bisect 1 32000 '@Ha:t03/1' jq -e '.[2] > 100000'
-
-   For special path elements see 'chain get' help
-
+   
+      lotus chain bisect [min height] [max height] '1/2/3/state/path' 'shell command' 'args'
+   
+      Returns the first tipset in which condition is true
+                     v
+      [start] FFFFFFFTTT [end]
+   
+      Example: find height at which deal ID 100 000 appeared
+       - lotus chain bisect 1 32000 '@Ha:t03/1' jq -e '.[2] > 100000'
+   
+      For special path elements see 'chain get' help
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -2382,8 +2379,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2415,8 +2411,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2453,7 +2448,6 @@ OPTIONS:
    --max-fee value  Spend up to X FIL per DisputeWindowedPoSt message
    --from value     optionally specify the account to send messages from
    --help, -h       show help (default: false)
-   --version, -v    print the version (default: false)
    
 ```
 
@@ -2499,8 +2493,7 @@ COMMANDS:
    help, h    Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2527,23 +2520,22 @@ USAGE:
 
 DESCRIPTION:
    Set the log level for logging systems:
-
-   The system flag can be specified multiple times.
-
-   eg) log set-level --system chain --system chainxchg debug
-
-   Available Levels:
-   debug
-   info
-   warn
-   error
-
-   Environment Variables:
-   GOLOG_LOG_LEVEL - Default log level for all log systems
-   GOLOG_LOG_FMT   - Change output log format (json, nocolor)
-   GOLOG_FILE      - Write logs to file
-   GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
-
+   
+      The system flag can be specified multiple times.
+   
+      eg) log set-level --system chain --system chainxchg debug
+   
+      Available Levels:
+      debug
+      info
+      warn
+      error
+   
+      Environment Variables:
+      GOLOG_LOG_LEVEL - Default log level for all log systems
+      GOLOG_LOG_FMT   - Change output log format (json, nocolor)
+      GOLOG_FILE      - Write logs to file
+      GOLOG_OUTPUT    - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr
 
 OPTIONS:
    --system value  limit to log system
@@ -2607,22 +2599,24 @@ USAGE:
    lotus net command [command options] [arguments...]
 
 COMMANDS:
-   peers         Print peers
-   connect       Connect to a peer
-   listen        List listen addresses
-   id            Get node identity
-   findpeer      Find the addresses of a given peerID
-   scores        Print peers' pubsub scores
-   reachability  Print information about reachability from the internet
-   bandwidth     Print bandwidth usage information
-   block         Manage network connection gating rules
-   stat          Report resource usage for a scope
-   limit         Get or set resource limits for a scope
-   help, h       Shows a list of commands or help for one command
+   peers           Print peers
+   connect         Connect to a peer
+   listen          List listen addresses
+   id              Get node identity
+   findpeer        Find the addresses of a given peerID
+   scores          Print peers' pubsub scores
+   reachability    Print information about reachability from the internet
+   bandwidth       Print bandwidth usage information
+   block           Manage network connection gating rules
+   stat            Report resource usage for a scope
+   limit           Get or set resource limits for a scope
+   protect         Add one or more peer IDs to the list of protected peer connections
+   unprotect       Remove one or more peer IDs from the list of protected peer connections.
+   list-protected  List the peer IDs with protected connection.
+   help, h         Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2750,8 +2744,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2770,8 +2763,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2829,8 +2821,7 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 
@@ -2896,15 +2887,14 @@ USAGE:
 
 DESCRIPTION:
    Report resource usage for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-  - all           -- reports the resource usage for all currently active scopes.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+     - all           -- reports the resource usage for all currently active scopes.
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -2921,16 +2911,15 @@ USAGE:
 
 DESCRIPTION:
    Get or set resource limits for a scope.
-
-  The scope can be one of the following:
-  - system        -- reports the system aggregate resource usage.
-  - transient     -- reports the transient resource usage.
-  - svc:<service> -- reports the resource usage of a specific service.
-  - proto:<proto> -- reports the resource usage of a specific protocol.
-  - peer:<peer>   -- reports the resource usage of a specific peer.
-
- The limit is json-formatted, with the same structure as the limits file.
-
+   
+     The scope can be one of the following:
+     - system        -- reports the system aggregate resource usage.
+     - transient     -- reports the transient resource usage.
+     - svc:<service> -- reports the resource usage of a specific service.
+     - proto:<proto> -- reports the resource usage of a specific protocol.
+     - peer:<peer>   -- reports the resource usage of a specific peer.
+   
+    The limit is json-formatted, with the same structure as the limits file.
 
 OPTIONS:
    --set       set the limit for a scope (default: false)
@@ -2938,6 +2927,45 @@ OPTIONS:
    
 ```
 
+### lotus net protect
+```
+NAME:
+   lotus net protect - Add one or more peer IDs to the list of protected peer connections
+
+USAGE:
+   lotus net protect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus net unprotect
+```
+NAME:
+   lotus net unprotect - Remove one or more peer IDs from the list of protected peer connections.
+
+USAGE:
+   lotus net unprotect [command options] <peer-id> [<peer-id>...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
+### lotus net list-protected
+```
+NAME:
+   lotus net list-protected - List the peer IDs with protected connection.
+
+USAGE:
+   lotus net list-protected [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ## lotus sync
 ```
 NAME:
@@ -2956,8 +2984,7 @@ COMMANDS:
    help, h     Shows a list of commands or help for one command
 
 OPTIONS:
-   --help, -h     show help (default: false)
-   --version, -v  print the version (default: false)
+   --help, -h  show help (default: false)
    
 ```
 

documentation/en/default-lotus-miner-config.toml

@@ -265,6 +265,27 @@
       #Path = ""
 
 
+[IndexProvider]
+  # env var: LOTUS_INDEXPROVIDER_LINKCACHESIZE
+  #LinkCacheSize = 1024
+
+  # env var: LOTUS_INDEXPROVIDER_LINKEDCHUNKSIZE
+  #LinkedChunkSize = 16384
+
+  # env var: LOTUS_INDEXPROVIDER_PUBSUBTOPIC
+  #PubSubTopic = "/indexer/ingest/mainnet"
+
+  # env var: LOTUS_INDEXPROVIDER_PURGELINKCACHE
+  #PurgeLinkCache = false
+
+  # env var: LOTUS_INDEXPROVIDER_PUBLISHERKIND
+  #PublisherKind = "dtsync"
+
+  [IndexProvider.HttpPublisher]
+    # env var: LOTUS_INDEXPROVIDER_HTTPPUBLISHER_LISTENMULTIADDR
+    #ListenMultiaddr = "/ip4/0.0.0.0/tcp/3104/http"
+
+
 [Sealing]
   # Upper bound on how many sectors can be waiting for more deals to be packed in it before it begins sealing at any given time.
   # If the miner is accepting multiple deals in parallel, up to MaxWaitDealsSectors of new sectors will be created.
@@ -438,6 +459,9 @@
   # env var: LOTUS_STORAGE_ALLOWPROVEREPLICAUPDATE2
   #AllowProveReplicaUpdate2 = true
 
+  # env var: LOTUS_STORAGE_ALLOWREGENSECTORKEY
+  #AllowRegenSectorKey = true
+
   # env var: LOTUS_STORAGE_RESOURCEFILTERING
   #ResourceFiltering = "hardware"
 

extern/sector-storage/faults.go

@@ -19,11 +19,11 @@ import (
 
 // FaultTracker TODO: Track things more actively
 type FaultTracker interface {
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error)
 }
 
 // CheckProvable returns unprovable sectors
-func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
+func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	var bad = make(map[abi.SectorID]string)
 
 	ssize, err := pp.SectorSize()
@@ -32,72 +32,76 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 	}
 
 	// TODO: More better checks
-	for _, sector := range sectors {
+	for i, sector := range sectors {
 		err := func() error {
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
+			var fReplica string
+			var fCache string
 
-			locked, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTNone)
-			if err != nil {
-				return xerrors.Errorf("acquiring sector lock: %w", err)
-			}
-
-			if !locked {
-				log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
-				bad[sector.ID] = fmt.Sprint("can't acquire read lock")
-				return nil
-			}
-
-			lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
-			if err != nil {
-				log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-				bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
-				return nil
-			}
-
-			// temporary hack to make the check work with snapdeals
-			// will go away in https://github.com/filecoin-project/lotus/pull/7971
-			if lp.Sealed == "" || lp.Cache == "" {
-				// maybe it's update
+			if update[i] {
 				lockedUpdate, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone)
 				if err != nil {
 					return xerrors.Errorf("acquiring sector lock: %w", err)
 				}
-				if lockedUpdate {
-					lp, _, err = m.localStore.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
-					if err != nil {
-						log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-						bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
-						return nil
-					}
-					lp.Sealed, lp.Cache = lp.Update, lp.UpdateCache
+				if !lockedUpdate {
+					log.Warnw("CheckProvable Sector FAULT: can't acquire read lock on update replica", "sector", sector)
+					bad[sector.ID] = fmt.Sprint("can't acquire read lock")
+					return nil
 				}
+				lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: acquire sector update replica in checkProvable", "sector", sector, "error", err)
+					bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
+					return nil
+				}
+				fReplica, fCache = lp.Update, lp.UpdateCache
+			} else {
+				locked, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTNone)
+				if err != nil {
+					return xerrors.Errorf("acquiring sector lock: %w", err)
+				}
+
+				if !locked {
+					log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
+					bad[sector.ID] = fmt.Sprint("can't acquire read lock")
+					return nil
+				}
+
+				lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
+					bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
+					return nil
+				}
+				fReplica, fCache = lp.Sealed, lp.Cache
+
 			}
 
-			if lp.Sealed == "" || lp.Cache == "" {
-				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
-				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed)
+			if fReplica == "" || fCache == "" {
+				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", fReplica, "cache", fCache)
+				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", fCache, fReplica)
 				return nil
 			}
 
 			toCheck := map[string]int64{
-				lp.Sealed:                        1,
-				filepath.Join(lp.Cache, "p_aux"): 0,
+				fReplica:                       1,
+				filepath.Join(fCache, "p_aux"): 0,
 			}
 
-			addCachePathsForSectorSize(toCheck, lp.Cache, ssize)
+			addCachePathsForSectorSize(toCheck, fCache, ssize)
 
 			for p, sz := range toCheck {
 				st, err := os.Stat(p)
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", fReplica, "cache", fCache, "file", p, "err", err)
 					bad[sector.ID] = fmt.Sprintf("%s", err)
 					return nil
 				}
 
 				if sz != 0 {
 					if st.Size() != int64(ssize)*sz {
-						log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
+						log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", fReplica, "cache", fCache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
 						bad[sector.ID] = fmt.Sprintf("%s is wrong size (got %d, expect %d)", p, st.Size(), int64(ssize)*sz)
 						return nil
 					}
@@ -118,14 +122,14 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 					sector.ID.Number,
 				})
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("generating fallback challenges: %s", err)
 					return nil
 				}
 
 				commr, err := rg(ctx, sector.ID)
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("getting commR: %s", err)
 					return nil
 				}
@@ -136,12 +140,12 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 						SectorNumber: sector.ID.Number,
 						SealedCID:    commr,
 					},
-					CacheDirPath:     lp.Cache,
+					CacheDirPath:     fCache,
 					PoStProofType:    wpp,
-					SealedSectorPath: lp.Sealed,
+					SealedSectorPath: fReplica,
 				}, ch.Challenges[sector.ID.Number])
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("generating vanilla proof: %s", err)
 					return nil
 				}

extern/sector-storage/manager.go

@@ -105,6 +105,7 @@ type SealerConfig struct {
 	AllowUnseal              bool
 	AllowReplicaUpdate       bool
 	AllowProveReplicaUpdate2 bool
+	AllowRegenSectorKey      bool
 
 	// ResourceFiltering instructs the system which resource filtering strategy
 	// to use when evaluating tasks against this worker. An empty value defaults
@@ -169,6 +170,9 @@ func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls store
 	if sc.AllowProveReplicaUpdate2 {
 		localTasks = append(localTasks, sealtasks.TTProveReplicaUpdate2)
 	}
+	if sc.AllowRegenSectorKey {
+		localTasks = append(localTasks, sealtasks.TTRegenSectorKey)
+	}
 
 	wcfg := WorkerConfig{
 		IgnoreResourceFiltering: sc.ResourceFiltering == ResourceFilteringDisabled,

extern/sector-storage/mock/mock.go

@@ -505,7 +505,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro
 	return nil
 }
 
-func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
+func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	bad := map[abi.SectorID]string{}
 
 	for _, sid := range ids {

extern/sector-storage/stores/http_handler.go

@@ -294,6 +294,10 @@ func ftFromString(t string) (storiface.SectorFileType, error) {
 		return storiface.FTSealed, nil
 	case storiface.FTCache.String():
 		return storiface.FTCache, nil
+	case storiface.FTUpdate.String():
+		return storiface.FTUpdate, nil
+	case storiface.FTUpdateCache.String():
+		return storiface.FTUpdateCache, nil
 	default:
 		return 0, xerrors.Errorf("unknown sector file type: '%s'", t)
 	}

extern/storage-sealing/checks.go

@@ -214,8 +214,13 @@ func checkReplicaUpdate(ctx context.Context, maddr address.Address, si SectorInf
 	if err != nil {
 		return &ErrApi{xerrors.Errorf("calling StateComputeDataCommitment: %w", err)}
 	}
-	if si.UpdateUnsealed == nil || !commD.Equals(*si.UpdateUnsealed) {
-		return &ErrBadRU{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)}
+
+	if si.UpdateUnsealed == nil {
+		return &ErrBadRU{xerrors.New("nil UpdateUnsealed cid after replica update")}
+	}
+
+	if !commD.Equals(*si.UpdateUnsealed) {
+		return &ErrBadRU{xerrors.Errorf("calculated CommD differs from updated replica: %s != %s", commD, *si.UpdateUnsealed)}
 	}
 
 	if si.UpdateSealed == nil {

extern/storage-sealing/upgrade_queue.go

@@ -108,7 +108,8 @@ func sectorActive(ctx context.Context, api SealingAPI, maddr address.Address, to
 	if err != nil {
 		return false, xerrors.Errorf("failed to check active sectors: %w", err)
 	}
-	// Check if sector is among active sectors
+
+	// Ensure the upgraded sector is active
 	var found bool
 	for _, si := range active {
 		if si.SectorNumber == sector {

gateway/node.go

@@ -51,6 +51,7 @@ type TargetAPI interface {
 	MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
 	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (api.MsigVesting, error)
 	MsigGetPending(ctx context.Context, addr address.Address, ts types.TipSetKey) ([]*api.MsigTransaction, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
@@ -282,6 +283,13 @@ func (gw *Node) MsigGetVested(ctx context.Context, addr address.Address, start t
 	return gw.target.MsigGetVested(ctx, addr, start, end)
 }
 
+func (gw *Node) MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MsigVesting, error) {
+	if err := gw.checkTipsetKey(ctx, tsk); err != nil {
+		return api.MsigVesting{}, err
+	}
+	return gw.target.MsigGetVestingSchedule(ctx, addr, tsk)
+}
+
 func (gw *Node) MsigGetPending(ctx context.Context, addr address.Address, tsk types.TipSetKey) ([]*api.MsigTransaction, error) {
 	if err := gw.checkTipsetKey(ctx, tsk); err != nil {
 		return nil, err

go.mod

@@ -2,6 +2,8 @@ module github.com/filecoin-project/lotus
 
 go 1.16
 
+retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
+
 require (
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0
 	github.com/BurntSushi/toml v0.4.1
@@ -26,7 +28,7 @@ require (
 	github.com/elastic/gosigar v0.14.1
 	github.com/etclabscore/go-openrpc-reflect v0.0.36
 	github.com/fatih/color v1.13.0
-	github.com/filecoin-project/dagstore v0.4.4
+	github.com/filecoin-project/dagstore v0.5.2
 	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
 	github.com/filecoin-project/go-address v0.0.6
 	github.com/filecoin-project/go-bitfield v0.2.4
@@ -36,21 +38,23 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.14.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.19.0
+	github.com/filecoin-project/go-fil-markets v1.19.2-0.20220223141410-2460e15e07a8
 	github.com/filecoin-project/go-jsonrpc v0.1.5
+	github.com/filecoin-project/go-legs v0.3.4
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
 	github.com/filecoin-project/go-state-types v0.1.3
 	github.com/filecoin-project/go-statemachine v1.0.1
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
+	github.com/filecoin-project/index-provider v0.3.1
 	github.com/filecoin-project/specs-actors v0.9.14
 	github.com/filecoin-project/specs-actors/v2 v2.3.6
 	github.com/filecoin-project/specs-actors/v3 v3.1.1
 	github.com/filecoin-project/specs-actors/v4 v4.0.1
 	github.com/filecoin-project/specs-actors/v5 v5.0.4
 	github.com/filecoin-project/specs-actors/v6 v6.0.1
-	github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1
+	github.com/filecoin-project/specs-actors/v7 v7.0.0
 	github.com/filecoin-project/specs-storage v0.2.0
 	github.com/filecoin-project/test-vectors/schema v0.0.5
 	github.com/gbrlsnchs/jwt/v3 v3.0.1
@@ -77,7 +81,7 @@ require (
 	github.com/ipfs/go-ds-badger2 v0.1.2
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ds-measure v0.2.0
-	github.com/ipfs/go-fs-lock v0.0.6
+	github.com/ipfs/go-fs-lock v0.0.7
 	github.com/ipfs/go-graphsync v0.12.0
 	github.com/ipfs/go-ipfs-blockstore v1.1.2
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
@@ -96,9 +100,10 @@ require (
 	github.com/ipfs/go-merkledag v0.5.1
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
-	github.com/ipfs/go-path v0.0.7
+	github.com/ipfs/go-path v0.2.1
 	github.com/ipfs/go-unixfs v0.3.1
-	github.com/ipfs/interface-go-ipfs-core v0.4.0
+	github.com/ipfs/go-unixfsnode v1.2.0
+	github.com/ipfs/interface-go-ipfs-core v0.5.2
 	github.com/ipld/go-car v0.3.3
 	github.com/ipld/go-car/v2 v2.1.1
 	github.com/ipld/go-codec-dagpb v1.3.0
@@ -108,7 +113,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-eventbus v0.2.1
-	github.com/libp2p/go-libp2p v0.18.0-rc4
+	github.com/libp2p/go-libp2p v0.18.0-rc5
 	github.com/libp2p/go-libp2p-connmgr v0.3.1 // indirect
 	github.com/libp2p/go-libp2p-core v0.14.0
 	github.com/libp2p/go-libp2p-discovery v0.6.0
@@ -118,9 +123,9 @@ require (
 	github.com/libp2p/go-libp2p-pubsub v0.6.1
 	github.com/libp2p/go-libp2p-quic-transport v0.16.1
 	github.com/libp2p/go-libp2p-record v0.1.3
-	github.com/libp2p/go-libp2p-resource-manager v0.1.3
+	github.com/libp2p/go-libp2p-resource-manager v0.1.4
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3
-	github.com/libp2p/go-libp2p-swarm v0.10.1
+	github.com/libp2p/go-libp2p-swarm v0.10.2
 	github.com/libp2p/go-libp2p-tls v0.3.1
 	github.com/libp2p/go-libp2p-yamux v0.8.2
 	github.com/libp2p/go-maddr-filter v0.1.0
@@ -143,9 +148,9 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
-	github.com/urfave/cli/v2 v2.2.0
+	github.com/urfave/cli/v2 v2.3.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8
+	github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325
@@ -155,11 +160,9 @@ require (
 	go.opentelemetry.io/otel/bridge/opencensus v0.25.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.2.0
 	go.opentelemetry.io/otel/sdk v1.2.0
-	go.uber.org/dig v1.10.0 // indirect
-	go.uber.org/fx v1.9.0
+	go.uber.org/fx v1.15.0
 	go.uber.org/multierr v1.7.0
 	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211209171907-798191bca915
@@ -168,9 +171,10 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools v2.2.0+incompatible
-	lukechampine.com/blake3 v1.1.7 // indirect
 )
 
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
+
+replace github.com/ipfs/go-path => github.com/ipfs/go-path v0.0.7

go.sum

@@ -1,3 +1,4 @@
+bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -87,6 +88,7 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o=
 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -96,6 +98,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -129,6 +132,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
 github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
@@ -159,6 +163,7 @@ github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRt
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -166,6 +171,7 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
 github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
@@ -198,6 +204,7 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -269,6 +276,7 @@ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6
 github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/gosigar v0.14.1 h1:T0aQ7n/n2ZA9W7DmAnj60v+qzqKERdBgJBO1CG2W6rc=
 github.com/elastic/gosigar v0.14.1/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE=
 github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -290,9 +298,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
 github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/filecoin-project/dagstore v0.4.3-0.20211211192320-72b849e131d2/go.mod h1:tlV8C11UljvFq3WWlMh2oMViEaVaPb6uT8eL/YQgDfk=
-github.com/filecoin-project/dagstore v0.4.4 h1:luolWahhzp3ulRsapGKE7raoLE3n2cFkQUJjPyqUmF4=
-github.com/filecoin-project/dagstore v0.4.4/go.mod h1:7BlOvaTJrFJ1Qemt5jHlLJ4VhDIuSIzGS0IwO/0AXPA=
+github.com/filecoin-project/dagstore v0.5.2 h1:Nd6oXdnolbbVhpMpkYT5PJHOjQp4OBSntHpMV5pxj3c=
+github.com/filecoin-project/dagstore v0.5.2/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0=
 github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
 github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
 github.com/filecoin-project/go-address v0.0.6 h1:DWQtj38ax+ogHwyH3VULRIoT8E6loyXqsk/p81xoY7M=
@@ -316,6 +323,7 @@ github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9AN
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o=
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
+github.com/filecoin-project/go-dagaggregator-unixfs v0.2.0/go.mod h1:WTuJWgBQY0omnQqa8kRPT9O0Uj5wQOgslVMUuTeHdJ8=
 github.com/filecoin-project/go-data-transfer v1.14.0 h1:4pnfJk8FYtqcdAg+QRGzaz57seUC/Tz+HJgPuGB7zdg=
 github.com/filecoin-project/go-data-transfer v1.14.0/go.mod h1:wNJKhaLLYBJDM3VFvgvYi4iUjPa69pz/1Q5Q4HzX2wE=
 github.com/filecoin-project/go-ds-versioning v0.0.0-20211206185234-508abd7c2aff/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4=
@@ -327,8 +335,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88Oq
 github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
-github.com/filecoin-project/go-fil-markets v1.19.0 h1:kap2q2wTM6tfkVO5gMA5DD9GUeTvkDhMfhjCtEwMDM8=
-github.com/filecoin-project/go-fil-markets v1.19.0/go.mod h1:qsb3apmo4RSJYCEq40QxVdU7UZospN6nFJLOBHuaIbc=
+github.com/filecoin-project/go-fil-markets v1.19.2-0.20220223141410-2460e15e07a8 h1:cuV4t78W8tUgmqvsc6T6qbWuycg7GzPcNsTdAOxrTns=
+github.com/filecoin-project/go-fil-markets v1.19.2-0.20220223141410-2460e15e07a8/go.mod h1:OeR49x+NPMfZMMgOresxzecMAgUB+lna2kwJABPwnt8=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -336,10 +344,13 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+
 github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI=
 github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI=
 github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g=
+github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43PjRCn3yBuUiZgIxuA0hTUamY=
 github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk=
 github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
+github.com/filecoin-project/go-legs v0.3.0/go.mod h1:x6nwM+DuN7NzlPndOoJuiHYCX+pze6+efPRx17nIA7M=
+github.com/filecoin-project/go-legs v0.3.4 h1:yid2IivTJ8JeF1ROA8jxNKNKJshU/HO1sRo5lEMNOUc=
+github.com/filecoin-project/go-legs v0.3.4/go.mod h1:P7ZPHqFG96OFaT11rnXyTdcxRdGkaH1dRyPxJSOiSuM=
 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak=
-github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
 github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs=
 github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
 github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8=
@@ -348,7 +359,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go
 github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
-github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0=
@@ -361,6 +371,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd
 github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo=
 github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus=
 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
+github.com/filecoin-project/index-provider v0.3.1 h1:ySGk2FnDbUEhq6QwakeU/p8tKnIoUm6JIpXPQ8RG1iM=
+github.com/filecoin-project/index-provider v0.3.1/go.mod h1:o8ineTLWnwtfBv7SnELpncza6SBU295esxOaXsTCl2I=
 github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
 github.com/filecoin-project/specs-actors v0.9.14 h1:68PVstg2UB3ZsMLF+DKFTAs/YKsqhKWynkr0IqmVRQY=
 github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
@@ -380,13 +392,16 @@ github.com/filecoin-project/specs-actors/v6 v6.0.1 h1:laxvHNsvrq83Y9n+W7znVCePi3
 github.com/filecoin-project/specs-actors/v6 v6.0.1/go.mod h1:V1AYfi5GkHXipx1mnVivoICZh3wtwPxDVuds+fbfQtk=
 github.com/filecoin-project/specs-actors/v7 v7.0.0-20211117170924-fd07a4c7dff9/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE=
 github.com/filecoin-project/specs-actors/v7 v7.0.0-20211222192039-c83bea50c402/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE=
-github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1 h1:FuDaXIbcw2hRsFI8SDTmsGGCE+NumpF6aiBoU/2X5W4=
-github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M=
+github.com/filecoin-project/specs-actors/v7 v7.0.0 h1:FQN7tjt3o68hfb3qLFSJBoLMuOFY0REkFVLO/zXj8RU=
+github.com/filecoin-project/specs-actors/v7 v7.0.0/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M=
 github.com/filecoin-project/specs-storage v0.2.0 h1:Y4UDv0apRQ3zI2GiPPubi8JblpUZZphEdaJUxCutfyg=
 github.com/filecoin-project/specs-storage v0.2.0/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU=
+github.com/filecoin-project/storetheindex v0.3.2 h1:KsTJer5Di/iapSrOWtbFTPMt5kS7Y3n3C9kOYH93gEA=
+github.com/filecoin-project/storetheindex v0.3.2/go.mod h1:d7y0FHAwPPSIdg+Oh7xV5B9TLebL7rvoKu591N/l2ZU=
 github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg=
 github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
 github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
 github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -399,8 +414,12 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
 github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
 github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
+github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs=
+github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0=
 github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4=
 github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
@@ -409,6 +428,7 @@ github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U
 github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -665,6 +685,8 @@ github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIc
 github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis=
 github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
 github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
+github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM=
+github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI=
 github.com/ipfs/go-bitswap v0.5.1 h1:721YAEDBnLIrvcIMkCHCdqp34hA8jwL9yKMkyJpSpco=
 github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo=
 github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
@@ -673,6 +695,8 @@ github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/d
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
 github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So=
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
+github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
+github.com/ipfs/go-blockservice v0.1.7/go.mod h1:GmS+BAt4hrwBKkzE11AFDQUrnvqjwFatGS2MY7wOjEM=
 github.com/ipfs/go-blockservice v0.2.1 h1:NJ4j/cwEfIg60rzAWcCIxRtOwbf6ZPK49MewNxObCPQ=
 github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -695,6 +719,7 @@ github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRV
 github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs=
 github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
@@ -712,6 +737,7 @@ github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1
 github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek=
 github.com/ipfs/go-ds-badger2 v0.1.2 h1:sQc2q1gaXrv8YFNeUtxil0neuyDf9hnVHfLsi7lpXfE=
 github.com/ipfs/go-ds-badger2 v0.1.2/go.mod h1:3FtQmDv6fMubygEfU43bsFelYpIiXX/XEYA54l9eCwg=
+github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
 github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
 github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
@@ -720,15 +746,25 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
 github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
 github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
+github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE=
+github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo=
+github.com/ipfs/go-filestore v0.1.0/go.mod h1:0KTrzoJnJ3sJDEDM09Vq8nz8H475rRyeq4i0n/bpF00=
 github.com/ipfs/go-filestore v1.1.0 h1:Pu4tLBi1bucu6/HU9llaOmb9yLFk/sgP+pW764zNDoE=
 github.com/ipfs/go-filestore v1.1.0/go.mod h1:6e1/5Y6NvLuCRdmda/KA4GUhXJQ3Uat6vcWm2DJfxc8=
-github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0=
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
+github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
+github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
+github.com/ipfs/go-graphsync v0.11.0/go.mod h1:wC+c8vGVjAHthsVIl8LKr37cUra2GOaMYcQNNmMxDqE=
 github.com/ipfs/go-graphsync v0.12.0 h1:QCsVHVzb9FTkcm3NEa8GjXnUeGit1L9s08HcSVQ4m/g=
 github.com/ipfs/go-graphsync v0.12.0/go.mod h1:nASYWYETgsnMbQ3+DirNImOHQ8TY0a5AhAqyOY55tUg=
+github.com/ipfs/go-ipfs v0.11.0/go.mod h1:g68Thu2Ho11AWoHsN34P5fSK7iA6OWWRy3T/g8HLixc=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
+github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
+github.com/ipfs/go-ipfs-blockstore v0.1.6/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
 github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
+github.com/ipfs/go-ipfs-blockstore v1.0.4-0.20210205083733-fb07d7bc5aec/go.mod h1:feuklK+m9POeWJzYQO7l05yNEgUiX5oELBNA8/Be33E=
+github.com/ipfs/go-ipfs-blockstore v1.0.4/go.mod h1:uL7/gTJ8QIZ3MtA3dWf+s1a0U3fJy2fcEZAsovpRp+w=
 github.com/ipfs/go-ipfs-blockstore v1.1.1/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
 github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw=
 github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
@@ -737,15 +773,18 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
-github.com/ipfs/go-ipfs-cmds v0.3.0 h1:mi9oYrSCox5aBhutqAYqw6/9crlyGbw4E/aJtwS4zI4=
 github.com/ipfs/go-ipfs-cmds v0.3.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
-github.com/ipfs/go-ipfs-config v0.5.3 h1:3GpI/xR9FoJNTjU6YvCMRbYyEi0dBVY5UtlUTcNRlSA=
+github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw=
+github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
 github.com/ipfs/go-ipfs-config v0.5.3/go.mod h1:nSLCFtlaL+2rbl3F+9D4gQZQbT1LjRKx7TJg/IHz6oM=
+github.com/ipfs/go-ipfs-config v0.18.0 h1:Ta1aNGNEq6RIvzbw7dqzCVZJKb7j+Dd35JFnAOCpT8g=
+github.com/ipfs/go-ipfs-config v0.18.0/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
 github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
+github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
 github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
 github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU=
 github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
@@ -761,11 +800,14 @@ github.com/ipfs/go-ipfs-files v0.0.9 h1:OFyOfmuVDu9c5YtjSDORmwXzE6fmZikzZpzsnNkg
 github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84=
 github.com/ipfs/go-ipfs-http-client v0.0.6 h1:k2QllZyP7Fz5hMgsX5hvHfn1WPG9Ngdy5WknQ7JNhBM=
 github.com/ipfs/go-ipfs-http-client v0.0.6/go.mod h1:8e2dQbntMZKxLfny+tyXJ7bJHZFERp/2vyzZdvkeLMc=
+github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
+github.com/ipfs/go-ipfs-pinner v0.2.1/go.mod h1:l1AtLL5bovb7opnG77sh4Y10waINz3Y1ni6CvTzx7oo=
 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
 github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
 github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
 github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY=
 github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
+github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8=
 github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
 github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
 github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY=
@@ -784,6 +826,7 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA=
 github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
+github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI=
 github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI=
 github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc=
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
@@ -814,39 +857,49 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M=
+github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUOg3ojR5MOE=
 github.com/ipfs/go-merkledag v0.5.1 h1:tr17GPP5XtPhvPPiWtu20tSGZiZDuTaJRXBLcr79Umk=
 github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
+github.com/ipfs/go-mfs v0.2.1/go.mod h1:Woj80iuw4ajDnIP6+seRaoHpPsc9hmL0pk/nDNDWP88=
+github.com/ipfs/go-namesys v0.4.0/go.mod h1:jpJwzodyP8DZdWN6DShRjVZw6gaqMr4nQLBSxU5cR6E=
 github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho=
 github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno=
 github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ=
 github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY=
 github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
 github.com/ipfs/go-peertaskqueue v0.7.1 h1:7PLjon3RZwRQMgOTvYccZ+mjzkmds/7YzSWKFlBAypE=
 github.com/ipfs/go-peertaskqueue v0.7.1/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
+github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4=
 github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4=
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
+github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8=
 github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
+github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4=
 github.com/ipfs/go-unixfsnode v1.2.0 h1:tHHBJftsJyHGa8bS62PpkYNqHy/Sug3c/vxxC8NaGQY=
 github.com/ipfs/go-unixfsnode v1.2.0/go.mod h1:mQEgLjxkV/1mohkC4p7taRRBYPBeXu97SA3YaerT2q0=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
-github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg=
 github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
+github.com/ipfs/interface-go-ipfs-core v0.5.2 h1:m1/5U+WpOK2ZE7Qzs5iIu80QM1ZA3aWYi2Ilwpi+tdg=
+github.com/ipfs/interface-go-ipfs-core v0.5.2/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE=
 github.com/ipfs/iptb v1.4.0 h1:YFYTrCkLMRwk/35IMyC6+yjoQSHTEcNcefBStLJzgvo=
 github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg=
 github.com/ipfs/iptb-plugins v0.3.0 h1:C1rpq1o5lUZtaAOkLIox5akh6ba4uk/3RwWc6ttVxw0=
 github.com/ipfs/iptb-plugins v0.3.0/go.mod h1:5QtOvckeIw4bY86gSH4fgh3p3gCSMn3FmIKr4gaBncA=
+github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM=
 github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g=
+github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE=
 github.com/ipld/go-car v0.3.3-0.20211210032800-e6f244225a16/go.mod h1:/wkKF4908ULT4dFIFIUZYcfjAnj+KFnJvlh8Hsz1FbQ=
 github.com/ipld/go-car v0.3.3 h1:D6y+jvg9h2ZSv7GLUMWUwg5VTLy1E7Ak+uQw5orOg3I=
 github.com/ipld/go-car v0.3.3/go.mod h1:/wkKF4908ULT4dFIFIUZYcfjAnj+KFnJvlh8Hsz1FbQ=
-github.com/ipld/go-car/v2 v2.1.1-0.20211211000942-be2525f6bf2d/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
 github.com/ipld/go-car/v2 v2.1.1 h1:saaKz4nC0AdfCGHLYKeXLGn8ivoPC54fyS55uyOLKwA=
 github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
 github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
@@ -857,7 +910,10 @@ github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/j
 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
+github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
 github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM=
+github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
+github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc=
 github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
@@ -866,6 +922,7 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
+github.com/ipld/go-storethehash v0.0.1/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
@@ -973,6 +1030,7 @@ github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5
 github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70=
 github.com/libp2p/go-conn-security-multistream v0.3.0 h1:9UCIKlBL1hC9u7nkMXpD1nkc/T53PKMAn3/k9ivBAVc=
 github.com/libp2p/go-conn-security-multistream v0.3.0/go.mod h1:EEP47t4fw/bTelVmEzIDqSe69hO/ip52xBEhZMLWAHM=
+github.com/libp2p/go-doh-resolver v0.3.1/go.mod h1:y5go1ZppAq9N2eppbX0xON01CyPBeUg2yS6BTssssog=
 github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
 github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
 github.com/libp2p/go-eventbus v0.2.1 h1:VanAdErQnpTioN2TowqNcOijf6YwhuODe4pPKSDpxGc=
@@ -990,13 +1048,16 @@ github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZk
 github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k=
 github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw=
 github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o=
+github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0=
+github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo=
+github.com/libp2p/go-libp2p v0.14.0/go.mod h1:dsQrWLAoIn+GkHPN/U+yypizkHiB9tnv79Os+kSgQ4Q=
 github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0=
 github.com/libp2p/go-libp2p v0.14.4/go.mod h1:EIRU0Of4J5S8rkockZM7eJp2S0UrCyi55m2kJVru3rM=
 github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4=
 github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw=
 github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8=
-github.com/libp2p/go-libp2p v0.18.0-rc4 h1:OUsSbeu7q+Ck/bV9wHDxFzb08ORqBupHhpCmRBhWrJ8=
-github.com/libp2p/go-libp2p v0.18.0-rc4/go.mod h1:wzmsk1ioOq9FGQys2BN5BIw4nugP6+R+CyW3JbPEbbs=
+github.com/libp2p/go-libp2p v0.18.0-rc5 h1:88wWDHb9nNo0vBNCupLde3OTnFAkugOCNkrDfl3ivK4=
+github.com/libp2p/go-libp2p v0.18.0-rc5/go.mod h1:aZPS5l84bDvCvP4jkyEUT/J6YOpUq33Fgqrs3K59mpI=
 github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
 github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E=
 github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I=
@@ -1006,6 +1067,7 @@ github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/
 github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI=
 github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI=
 github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A=
+github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk=
 github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk=
 github.com/libp2p/go-libp2p-autonat v0.6.0/go.mod h1:bFC6kY8jwzNNWoqc8iGE57vsfwyJ/lP4O4DOV1e0B2o=
 github.com/libp2p/go-libp2p-autonat v0.7.0/go.mod h1:uPvPn6J7cN+LCfFwW5tpOYvAz5NvPTc4iBamTV/WDMg=
@@ -1078,8 +1140,12 @@ github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQO
 github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug=
 github.com/libp2p/go-libp2p-discovery v0.6.0 h1:1XdPmhMJr8Tmj/yUfkJMIi8mgwWrLUsCB3bMxdT+DSo=
 github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8=
+github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc=
+github.com/libp2p/go-libp2p-gostream v0.3.1 h1:XlwohsPn6uopGluEWs1Csv1QCEjrTXf2ZQagzZ5paAg=
+github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI=
 github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go=
 github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8=
+github.com/libp2p/go-libp2p-http v0.2.1/go.mod h1:9KdioZ7XqNH0eZkZG9bulZLzHv11A7/12fT97agqWhg=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
 github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
@@ -1100,6 +1166,7 @@ github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3
 github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
 github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo=
 github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek=
+github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs=
 github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw=
 github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g=
 github.com/libp2p/go-libp2p-mplex v0.5.0 h1:vt3k4E4HSND9XH4Z8rUpacPJFSAgLOv6HDvG8W9Ks9E=
@@ -1114,6 +1181,7 @@ github.com/libp2p/go-libp2p-net v0.0.2/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8
 github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q=
 github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ=
 github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
+github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM=
 github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q=
 github.com/libp2p/go-libp2p-noise v0.3.0 h1:NCVH7evhVt9njbTQshzT7N1S3Q6fjj9M11FCgfH5+cA=
 github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ=
@@ -1142,6 +1210,7 @@ github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uz
 github.com/libp2p/go-libp2p-pubsub v0.6.0/go.mod h1:nJv87QM2cU0w45KPR1rZicq+FmFIOD16zmT+ep1nOmg=
 github.com/libp2p/go-libp2p-pubsub v0.6.1 h1:wycbV+f4rreCoVY61Do6g/BUk0RIrbNRcYVbn+QkjGk=
 github.com/libp2p/go-libp2p-pubsub v0.6.1/go.mod h1:nJv87QM2cU0w45KPR1rZicq+FmFIOD16zmT+ep1nOmg=
+github.com/libp2p/go-libp2p-pubsub-router v0.5.0/go.mod h1:TRJKskSem3C0aSb3CmRgPwq6IleVFzds6hS09fmZbGM=
 github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU=
 github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA=
 github.com/libp2p/go-libp2p-quic-transport v0.11.2/go.mod h1:wlanzKtIh6pHrq+0U3p3DY9PJfGqxMgPaGKaK5LifwQ=
@@ -1158,8 +1227,9 @@ github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGd
 github.com/libp2p/go-libp2p-record v0.1.3 h1:R27hoScIhQf/A8XJZ8lYpnqh9LatJ5YbHs28kCIfql0=
 github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ6EMg7+/vv2+9pY4=
 github.com/libp2p/go-libp2p-resource-manager v0.1.0/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y=
-github.com/libp2p/go-libp2p-resource-manager v0.1.3 h1:Umf0tW6WNXSb6Uoma0YT56azB5iikL/aeGAP7s7+f5o=
 github.com/libp2p/go-libp2p-resource-manager v0.1.3/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y=
+github.com/libp2p/go-libp2p-resource-manager v0.1.4 h1:RcxMD0pytOUimx3BqTVs6IqItb3H5Qg44SD7XyT68lw=
+github.com/libp2p/go-libp2p-resource-manager v0.1.4/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y=
 github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys=
 github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE=
 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY=
@@ -1176,13 +1246,15 @@ github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaT
 github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM=
 github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM=
 github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
+github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk=
+github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw=
 github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4=
 github.com/libp2p/go-libp2p-swarm v0.5.3/go.mod h1:NBn7eNW2lu568L7Ns9wdFrOhgRlkRnIDg0FLKbuu3i8=
 github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc=
 github.com/libp2p/go-libp2p-swarm v0.9.0/go.mod h1:2f8d8uxTJmpeqHF/1ujjdXZp+98nNIbujVOMEZxCbZ8=
 github.com/libp2p/go-libp2p-swarm v0.10.0/go.mod h1:71ceMcV6Rg/0rIQ97rsZWMzto1l9LnNquef+efcRbmA=
-github.com/libp2p/go-libp2p-swarm v0.10.1 h1:lXW3pgGt+BVmkzcFX61erX7l6Lt+WAamNhwa2Kf3eJM=
-github.com/libp2p/go-libp2p-swarm v0.10.1/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs=
+github.com/libp2p/go-libp2p-swarm v0.10.2 h1:UaXf+CTq6Ns1N2V1EgqJ9Q3xaRsiN7ImVlDMpirMAWw=
+github.com/libp2p/go-libp2p-swarm v0.10.2/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs=
 github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
 github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
 github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
@@ -1207,6 +1279,7 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2
 github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
 github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns=
 github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o=
+github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.3/go.mod h1:bpkldbOWXMrXhpZbSV1mQxTrefOg2Fi+k1ClDSA4ppw=
 github.com/libp2p/go-libp2p-transport-upgrader v0.4.6/go.mod h1:JE0WQuQdy+uLZ5zOaI3Nw9dWGYJIA7mywEtP2lMvnyk=
@@ -1226,6 +1299,8 @@ github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhL
 github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4=
 github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30=
 github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po=
+github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4=
+github.com/libp2p/go-libp2p-yamux v0.5.3/go.mod h1:Vy3TMonBAfTMXHWopsMc8iX/XGRYrRlpUaMzaeuHV/s=
 github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE=
 github.com/libp2p/go-libp2p-yamux v0.6.0/go.mod h1:MRhd6mAYnFRnSISp4M8i0ClV/j+mWHo2mYLifWGw33k=
 github.com/libp2p/go-libp2p-yamux v0.7.0/go.mod h1:fMyA0CsPfHkIuBU0wjRGrCjTBFiXTXxG0k5M4ETv+08=
@@ -1283,6 +1358,7 @@ github.com/libp2p/go-reuseport-transport v0.1.0/go.mod h1:vev0C0uMkzriDY59yFHD9v
 github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
+github.com/libp2p/go-socket-activation v0.1.0/go.mod h1:gzda2dNkMG5Ti2OfWNNwW0FDIbj0g/aJJU320FcLfhk=
 github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
 github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ=
 github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw=
@@ -1294,12 +1370,14 @@ github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19
 github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
 github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
 github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0=
+github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M=
 github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU=
 github.com/libp2p/go-tcp-transport v0.2.4/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU=
 github.com/libp2p/go-tcp-transport v0.2.7/go.mod h1:lue9p1b3VmZj1MhhEGB/etmvF/nBQ0X9CW2DutBT3MM=
 github.com/libp2p/go-tcp-transport v0.4.0/go.mod h1:0y52Rwrn4076xdJYu/51/qJIdxz+EWDAOG2S45sV3VI=
-github.com/libp2p/go-tcp-transport v0.5.0 h1:3ZPW8HAuyRAuFzyabE0hSrCXKKSWzROnZZX7DtcIatY=
 github.com/libp2p/go-tcp-transport v0.5.0/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y=
+github.com/libp2p/go-tcp-transport v0.5.1 h1:edOOs688VLZAozWC7Kj5/6HHXKNwi9M6wgRmmLa8M6Q=
+github.com/libp2p/go-tcp-transport v0.5.1/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y=
 github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I=
 github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc=
 github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9tnZjoFZnxykuaXU=
@@ -1307,6 +1385,7 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw
 github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y=
 github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM=
 github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
+github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk=
 github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA=
 github.com/libp2p/go-ws-transport v0.5.0/go.mod h1:I2juo1dNTbl8BKSBYo98XY85kU2xds1iamArLvl8kNg=
 github.com/libp2p/go-ws-transport v0.6.0 h1:326XBL6Q+5CQ2KtjXz32+eGu02W/Kz2+Fm4SpXdr0q4=
@@ -1321,6 +1400,8 @@ github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h
 github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
 github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI=
 github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
+github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U=
+github.com/libp2p/go-yamux/v2 v2.1.1/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ=
 github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ=
 github.com/libp2p/go-yamux/v2 v2.3.0/go.mod h1:iTU+lOIn/2h0AgKcL49clNTwfEw+WSfDYrXe05EyKIs=
 github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo=
@@ -1385,6 +1466,7 @@ github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
@@ -1403,6 +1485,7 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -1470,6 +1553,7 @@ github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/94
 github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.1.0/go.mod h1:01k2RAqtoXIuPa3DCavAE9/6jc6nM0H3EgZyfUhN2oY=
 github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
+github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
 github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
@@ -1489,8 +1573,9 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw
 github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
-github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c h1:VyANTtZ0wsx0IAZnCZhfMmAmfUyzJq/5JQi2hHOtKS0=
 github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
+github.com/multiformats/go-multicodec v0.4.0 h1:fbqb6ky7erjdD+/zaEBJgZWu1i8D6i/wmPywGK7sdow=
+github.com/multiformats/go-multicodec v0.4.0/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
@@ -1499,12 +1584,14 @@ github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpK
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
+github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag=
 github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA=
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38=
+github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k=
 github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k=
 github.com/multiformats/go-multistream v0.2.2 h1:TCYu1BHTDr1F/Qm75qwYISQdzGcRdC21nFgQW7l7GBo=
 github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs=
@@ -1787,6 +1874,7 @@ github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
@@ -1800,8 +1888,9 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
@@ -1809,8 +1898,9 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
-github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA=
 github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
+github.com/warpfork/go-testmark v0.9.0 h1:nc+uaCiv5lFQLYjhuC2LTYeJ7JaC+gdDmsz9r0ISy0Y=
+github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w=
@@ -1838,8 +1928,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8 h1:TEv7MId88TyIqIUL4hbf9otOookIolMxlEbN0ro671Y=
-github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008 h1:7WtW9D9VGpmRLuQmrPy2JobUNdka95z3MKEVpELtOjo=
+github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g=
@@ -1848,6 +1938,7 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
 github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
+github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso=
 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 h1:NwiwjQDB3CzQ5XH0rdMh1oQqzJH7O2PSLWxif/w3zsY=
 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4/go.mod h1:K+EVq8d5QcQ2At5VECsA+SNZvWefyBXh8TnIsxo1OvQ=
 github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
@@ -1869,6 +1960,7 @@ github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/
 github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8=
 github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1941,14 +2033,15 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
-go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
-go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
-go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
+go.uber.org/dig v1.12.0 h1:l1GQeZpEbss0/M4l/ZotuBndCrkMdjnygzgcuOjAdaY=
+go.uber.org/dig v1.12.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
+go.uber.org/fx v1.15.0 h1:kcfBpAm98n0ksanyyZLFE/Q3T7yPi13Ge2liu3TxR+A=
+go.uber.org/fx v1.15.0/go.mod h1:jI3RazQUhGv5KkpZIRv+kuP4CcgX3fnc0qX8bLnzbx8=
 go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
@@ -1998,6 +2091,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -2013,6 +2107,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2119,6 +2215,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -2198,6 +2295,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191206220618-eeba5f6aabab/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -2247,7 +2345,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915 h1:P+8mCzuEpyszAT6T42q0sxU+eveBAF/cJ2Kp0x6/8+0=
 golang.org/x/sys v0.0.0-20211209171907-798191bca915/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -2472,6 +2572,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
 gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -2521,6 +2622,7 @@ modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc=
 modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

itests/deals_anycid_test.go

@@ -0,0 +1,162 @@
+package itests
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipld/go-car"
+	"github.com/ipld/go-car/v2/blockstore"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/modules"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+func TestDealRetrieveByAnyCid(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
+
+	ctx := context.Background()
+	oldDelay := policy.GetPreCommitChallengeDelay()
+	policy.SetPreCommitChallengeDelay(5)
+	t.Cleanup(func() {
+		policy.SetPreCommitChallengeDelay(oldDelay)
+	})
+
+	// Allow 8MB sectors
+	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1)
+	kit.QuietMiningLogs()
+
+	// For these tests where the block time is artificially short, just use
+	// a deal start epoch that is guaranteed to be far enough in the future
+	// so that the deal starts sealing in time
+	startEpoch := abi.ChainEpoch(2 << 12)
+
+	// Override the dependency injection for the blockstore accessor, so that
+	// we can get a reference to the blockstore containing our deal later in
+	// the test
+	var bsa storagemarket.BlockstoreAccessor
+	bsaFn := func(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor {
+		bsa = modules.StorageBlockstoreAccessor(importmgr)
+		return bsa
+	}
+	bsaOpt := kit.ConstructorOpts(node.Override(new(storagemarket.BlockstoreAccessor), bsaFn))
+
+	// Allow 8MB sectors
+	eightMBSectorsOpt := kit.SectorSize(8 << 20)
+
+	// Create a client, and a miner with its own full node
+	_, client, miner, ens := kit.EnsembleTwoOne(t, kit.MockProofs(), bsaOpt, eightMBSectorsOpt)
+	ens.InterconnectAll().BeginMining(250 * time.Millisecond)
+
+	dh := kit.NewDealHarness(t, client, miner, miner)
+
+	// Generate a DAG with multiple levels, so that we can test the case where
+	// the client requests a CID for a block which is not the root block but
+	// does have a subtree below it in the DAG
+	dagOpts := kit.GeneratedDAGOpts{
+		// Max size of a block
+		ChunkSize: 1024,
+		// Max links from a block to other blocks
+		Maxlinks: 10,
+	}
+	carv1FilePath, _ := kit.CreateRandomCARv1(t, 5, 100*1024, dagOpts)
+	res, err := client.ClientImport(ctx, api.FileRef{Path: carv1FilePath, IsCAR: true})
+	require.NoError(t, err)
+
+	// Get the blockstore for the file
+	bs, err := bsa.Get(res.Root)
+	require.NoError(t, err)
+
+	// Get all CIDs from the file
+	sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: res.Root, Selector: selectorparse.CommonSelector_ExploreAllRecursively}})
+	prepared, err := sc.Prepare()
+	require.NoError(t, err)
+	cids := prepared.Cids()
+	for i, c := range cids {
+		blk, err := bs.Get(ctx, c)
+		require.NoError(t, err)
+
+		nd, err := ipld.Decode(blk)
+		require.NoError(t, err)
+
+		t.Log(i, c, len(nd.Links()))
+	}
+
+	// Create a storage deal
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = res.Root
+	dp.DealStartEpoch = startEpoch
+	dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price
+	dealCid := dh.StartDeal(ctx, dp)
+
+	// Wait for the deal to be sealed
+	dh.WaitDealSealed(ctx, dealCid, false, false, nil)
+
+	ask, err := miner.MarketGetRetrievalAsk(ctx)
+	require.NoError(t, err)
+	ask.PricePerByte = abi.NewTokenAmount(0)
+	ask.UnsealPrice = abi.NewTokenAmount(0)
+	err = miner.MarketSetRetrievalAsk(ctx, ask)
+	require.NoError(t, err)
+
+	// Fetch the deal data
+	info, err := client.ClientGetDealInfo(ctx, *dealCid)
+	require.NoError(t, err)
+
+	// Make retrievals against CIDs at different levels in the DAG
+	cidIndices := []int{1, 11, 27, 32, 47}
+	for _, val := range cidIndices {
+		t.Logf("performing retrieval for cid at index %d", val)
+
+		targetCid := cids[val]
+		offer, err := client.ClientMinerQueryOffer(ctx, miner.ActorAddr, targetCid, &info.PieceCID)
+		require.NoError(t, err)
+		require.Empty(t, offer.Err)
+
+		// retrieve in a CAR file and ensure roots match
+		outputCar := dh.PerformRetrieval(ctx, dealCid, targetCid, true, offer)
+		_, err = os.Stat(outputCar)
+		require.NoError(t, err)
+		f, err := os.Open(outputCar)
+		require.NoError(t, err)
+		ch, err := car.ReadHeader(bufio.NewReader(f))
+		require.NoError(t, err)
+		require.EqualValues(t, ch.Roots[0], targetCid)
+		require.NoError(t, f.Close())
+
+		// create CAR from original file starting at targetCid and ensure it matches the retrieved CAR file.
+		tmp, err := os.CreateTemp(t.TempDir(), "randcarv1")
+		require.NoError(t, err)
+		rd, err := blockstore.OpenReadOnly(carv1FilePath, blockstore.UseWholeCIDs(true))
+		require.NoError(t, err)
+		err = car.NewSelectiveCar(
+			ctx,
+			rd,
+			[]car.Dag{{
+				Root:     targetCid,
+				Selector: selectorparse.CommonSelector_ExploreAllRecursively,
+			}},
+		).Write(tmp)
+		require.NoError(t, err)
+		require.NoError(t, tmp.Close())
+		require.NoError(t, rd.Close())
+
+		kit.AssertFilesEqual(t, tmp.Name(), outputCar)
+		t.Log("car files match")
+	}
+}

itests/deals_concurrent_test.go

@@ -8,13 +8,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/stretchr/testify/require"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-fil-markets/shared_testutil"
 	"github.com/filecoin-project/go-state-types/abi"
+	provider "github.com/filecoin-project/index-provider"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
@@ -47,7 +49,9 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
 	runTest := func(t *testing.T, n int, fastRetrieval bool, carExport bool) {
 		api.RunningNodeType = api.NodeMiner // TODO(anteva): fix me
 
-		client, main, market, _ := kit.EnsembleWithMinerAndMarketNodes(t, kit.ThroughRPC())
+		idxProv := shared_testutil.NewMockIndexProvider()
+		idxProvOpt := kit.ConstructorOpts(node.Override(new(provider.Interface), idxProv))
+		client, main, market, _ := kit.EnsembleWithMinerAndMarketNodes(t, kit.ThroughRPC(), idxProvOpt)
 
 		dh := kit.NewDealHarness(t, client, main, market)
 
@@ -56,6 +60,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) {
 			FastRetrieval: fastRetrieval,
 			CarExport:     carExport,
 			StartEpoch:    startEpoch,
+			IndexProvider: idxProv,
 		})
 	}
 

itests/kit/blockminer.go

@@ -3,6 +3,7 @@ package kit
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -10,6 +11,7 @@ import (
 
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/filecoin-project/lotus/api"
 	aminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -63,11 +65,10 @@ func (p *partitionTracker) done(t *testing.T) bool {
 	return uint64(len(p.partitions)) == p.count(t)
 }
 
-func (p *partitionTracker) recordIfPost(t *testing.T, bm *BlockMiner, smsg *types.SignedMessage) (ret bool) {
+func (p *partitionTracker) recordIfPost(t *testing.T, bm *BlockMiner, msg *types.Message) (ret bool) {
 	defer func() {
 		ret = p.done(t)
 	}()
-	msg := smsg.Message
 	if !(msg.To == bm.miner.ActorAddr) {
 		return
 	}
@@ -82,11 +83,69 @@ func (p *partitionTracker) recordIfPost(t *testing.T, bm *BlockMiner, smsg *type
 	return
 }
 
+func (bm *BlockMiner) forcePoSt(ctx context.Context, ts *types.TipSet, dlinfo *dline.Info) {
+
+	tracker := newPartitionTracker(ctx, dlinfo.Index, bm)
+	if !tracker.done(bm.t) { // need to wait for post
+		bm.t.Logf("expect %d partitions proved but only see %d", len(tracker.partitions), tracker.count(bm.t))
+		poolEvts, err := bm.miner.FullNode.MpoolSub(ctx) //subscribe before checking pending so we don't miss any events
+		require.NoError(bm.t, err)
+
+		// First check pending messages we'll mine this epoch
+		msgs, err := bm.miner.FullNode.MpoolPending(ctx, types.EmptyTSK)
+		require.NoError(bm.t, err)
+		for _, msg := range msgs {
+			if tracker.recordIfPost(bm.t, bm, &msg.Message) {
+				fmt.Printf("found post in mempool pending\n")
+			}
+		}
+
+		// Account for included but not yet executed messages
+		for _, bc := range ts.Cids() {
+			msgs, err := bm.miner.FullNode.ChainGetBlockMessages(ctx, bc)
+			require.NoError(bm.t, err)
+			for _, msg := range msgs.BlsMessages {
+				if tracker.recordIfPost(bm.t, bm, msg) {
+					fmt.Printf("found post in message of prev tipset\n")
+				}
+
+			}
+			for _, msg := range msgs.SecpkMessages {
+				if tracker.recordIfPost(bm.t, bm, &msg.Message) {
+					fmt.Printf("found post in message of prev tipset\n")
+				}
+			}
+		}
+
+		// post not yet in mpool, wait for it
+		if !tracker.done(bm.t) {
+			bm.t.Logf("post missing from mpool, block mining suspended until it arrives")
+		POOL:
+			for {
+				bm.t.Logf("mpool event wait loop at block height %d, ts: %s", ts.Height(), ts.Key())
+				select {
+				case <-ctx.Done():
+					return
+				case evt := <-poolEvts:
+					bm.t.Logf("pool event: %d", evt.Type)
+					if evt.Type == api.MpoolAdd {
+						bm.t.Logf("incoming message %v", evt.Message)
+						if tracker.recordIfPost(bm.t, bm, &evt.Message.Message) {
+							fmt.Printf("found post in mempool evt\n")
+							break POOL
+						}
+					}
+				}
+			}
+			bm.t.Logf("done waiting on mpool")
+		}
+	}
+}
+
 // Like MineBlocks but refuses to mine until the window post scheduler has wdpost messages in the mempool
 // and everything shuts down if a post fails.  It also enforces that every block mined succeeds
 func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Duration) {
-
-	time.Sleep(3 * time.Second)
+	time.Sleep(time.Second)
 
 	// wrap context in a cancellable context.
 	ctx, bm.cancel = context.WithCancel(ctx)
@@ -94,8 +153,6 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 	go func() {
 		defer bm.wg.Done()
 
-		activeDeadlines := make(map[int]struct{})
-		_ = activeDeadlines
 		ts, err := bm.miner.FullNode.ChainHead(ctx)
 		require.NoError(bm.t, err)
 		wait := make(chan bool)
@@ -103,7 +160,7 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 		require.NoError(bm.t, err)
 		// read current out
 		curr := <-chg
-		require.Equal(bm.t, ts.Height(), curr[0].Val.Height())
+		require.Equal(bm.t, ts.Height(), curr[0].Val.Height(), "failed sanity check: are multiple miners mining with must post?")
 		for {
 			select {
 			case <-time.After(blocktime):
@@ -111,52 +168,15 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 				return
 			}
 			nulls := atomic.SwapInt64(&bm.nextNulls, 0)
-			require.Equal(bm.t, int64(0), nulls, "Injecting > 0 null blocks while `MustPost` mining is currently unsupported")
 
 			// Wake up and figure out if we are at the end of an active deadline
 			ts, err := bm.miner.FullNode.ChainHead(ctx)
 			require.NoError(bm.t, err)
-			tsk := ts.Key()
 
-			dlinfo, err := bm.miner.FullNode.StateMinerProvingDeadline(ctx, bm.miner.ActorAddr, tsk)
+			dlinfo, err := bm.miner.FullNode.StateMinerProvingDeadline(ctx, bm.miner.ActorAddr, ts.Key())
 			require.NoError(bm.t, err)
-			if ts.Height()+1 == dlinfo.Last() { // Last epoch in dline, we need to check that miner has posted
-
-				tracker := newPartitionTracker(ctx, dlinfo.Index, bm)
-				if !tracker.done(bm.t) { // need to wait for post
-					bm.t.Logf("expect %d partitions proved but only see %d", len(tracker.partitions), tracker.count(bm.t))
-					poolEvts, err := bm.miner.FullNode.MpoolSub(ctx)
-					require.NoError(bm.t, err)
-
-					// First check pending messages we'll mine this epoch
-					msgs, err := bm.miner.FullNode.MpoolPending(ctx, types.EmptyTSK)
-					require.NoError(bm.t, err)
-					for _, msg := range msgs {
-						tracker.recordIfPost(bm.t, bm, msg)
-					}
-
-					// post not yet in mpool, wait for it
-					if !tracker.done(bm.t) {
-						bm.t.Logf("post missing from mpool, block mining suspended until it arrives")
-					POOL:
-						for {
-							bm.t.Logf("mpool event wait loop at block height %d, ts: %s", ts.Height(), ts.Key())
-							select {
-							case <-ctx.Done():
-								return
-							case evt := <-poolEvts:
-								bm.t.Logf("pool event: %d", evt.Type)
-								if evt.Type == api.MpoolAdd {
-									bm.t.Logf("incoming message %v", evt.Message)
-									if tracker.recordIfPost(bm.t, bm, evt.Message) {
-										break POOL
-									}
-								}
-							}
-						}
-						bm.t.Logf("done waiting on mpool")
-					}
-				}
+			if ts.Height()+1+abi.ChainEpoch(nulls) >= dlinfo.Last() { // Next block brings us past the last epoch in dline, we need to wait for miner to post
+				bm.forcePoSt(ctx, ts, dlinfo)
 			}
 
 			var target abi.ChainEpoch
@@ -173,6 +193,12 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 					Done:        reportSuccessFn,
 				})
 				success = <-wait
+				if !success {
+					// if we are mining a new null block and it brings us past deadline boundary we need to wait for miner to post
+					if ts.Height()+1+abi.ChainEpoch(nulls+i) >= dlinfo.Last() {
+						bm.forcePoSt(ctx, ts, dlinfo)
+					}
+				}
 			}
 
 			// Wait until it shows up on the given full nodes ChainHead

itests/kit/circuit.go

@@ -0,0 +1,34 @@
+package kit
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+/*
+CircuitBreaker implements a simple time-based circuit breaker used for waiting for async operations to finish.
+
+This is how it works:
+	- It runs the `cb` function until it returns true,
+	- waiting for `throttle` duration between each iteration,
+	- or at most `timeout` duration until it breaks test execution.
+
+You can use it if t.Deadline() is not "granular" enough, and you want to know which specific piece of code timed out,
+or you need to set different deadlines in the same test.
+*/
+func CircuitBreaker(t *testing.T, label string, throttle, timeout time.Duration, cb func() bool) {
+	tmo := time.After(timeout)
+	for {
+		if cb() {
+			break
+		}
+		select {
+		case <-tmo:
+			t.Fatal("timeout: ", label)
+		default:
+			fmt.Printf("waiting: %s\n", label)
+			time.Sleep(throttle)
+		}
+	}
+}

itests/kit/client.go

@@ -111,9 +111,14 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode *TestFullNode)
 	tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-client")
 	require.NoError(t, err)
 	path := filepath.Join(tmpdir, "outfile.dat")
-	out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path)
-	fmt.Println("retrieve:\n", out)
-	require.Regexp(t, regexp.MustCompile("Success"), out)
+
+	for {
+		out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path)
+		fmt.Println("retrieve:\n", out)
+		if strings.Contains(out, "Success") {
+			break
+		}
+	}
 }
 
 func CreateImportFile(ctx context.Context, client api.FullNode, rseed int, size int) (res *api.ImportRes, path string, data []byte, err error) {

itests/kit/deals.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/shared_testutil"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
@@ -305,14 +306,20 @@ func (dh *DealHarness) StartSealingWaiting(ctx context.Context) {
 	}
 }
 
-func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool) (path string) {
-	// perform retrieval.
-	info, err := dh.client.ClientGetDealInfo(ctx, *deal)
-	require.NoError(dh.t, err)
+func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root cid.Cid, carExport bool, offers ...api.QueryOffer) (path string) {
+	var offer api.QueryOffer
+	if len(offers) == 0 {
+		// perform retrieval.
+		info, err := dh.client.ClientGetDealInfo(ctx, *deal)
+		require.NoError(dh.t, err)
 
-	offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID)
-	require.NoError(dh.t, err)
-	require.NotEmpty(dh.t, offers, "no offers")
+		offers, err := dh.client.ClientFindData(ctx, root, &info.PieceCID)
+		require.NoError(dh.t, err)
+		require.NotEmpty(dh.t, offers, "no offers")
+		offer = offers[0]
+	} else {
+		offer = offers[0]
+	}
 
 	carFile, err := ioutil.TempFile(dh.t.TempDir(), "ret-car")
 	require.NoError(dh.t, err)
@@ -326,7 +333,7 @@ func (dh *DealHarness) PerformRetrieval(ctx context.Context, deal *cid.Cid, root
 	updates, err := dh.client.ClientGetRetrievalUpdates(updatesCtx)
 	require.NoError(dh.t, err)
 
-	retrievalRes, err := dh.client.ClientRetrieve(ctx, offers[0].Order(caddr))
+	retrievalRes, err := dh.client.ClientRetrieve(ctx, offer.Order(caddr))
 	require.NoError(dh.t, err)
 consumeEvents:
 	for {
@@ -363,11 +370,6 @@ consumeEvents:
 		}))
 
 	ret := carFile.Name()
-	if carExport {
-		actualFile := dh.ExtractFileFromCAR(ctx, carFile)
-		ret = actualFile.Name()
-		_ = actualFile.Close() //nolint:errcheck
-	}
 
 	return ret
 }
@@ -404,9 +406,11 @@ type RunConcurrentDealsOpts struct {
 	CarExport                bool
 	StartEpoch               abi.ChainEpoch
 	UseCARFileForStorageDeal bool
+	IndexProvider            *shared_testutil.MockIndexProvider
 }
 
 func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
+	ctx := context.Background()
 	errgrp, _ := errgroup.WithContext(context.Background())
 	for i := 0; i < opts.N; i++ {
 		i := i
@@ -429,10 +433,28 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) {
 				UseCARFileForStorageDeal: opts.UseCARFileForStorageDeal,
 			})
 
+			// Check that the storage provider announced the deal to indexers
+			if opts.IndexProvider != nil {
+				notifs := opts.IndexProvider.GetNotifs()
+				_, ok := notifs[string(deal.Bytes())]
+				require.True(dh.t, ok)
+			}
+
 			dh.t.Logf("retrieving deal %d/%d", i, opts.N)
 
 			outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, opts.CarExport)
-			AssertFilesEqual(dh.t, inPath, outPath)
+
+			if opts.CarExport {
+				f, err := os.Open(outPath)
+				require.NoError(dh.t, err)
+				actualFile := dh.ExtractFileFromCAR(ctx, f)
+				require.NoError(dh.t, f.Close())
+
+				AssertFilesEqual(dh.t, inPath, actualFile.Name())
+			} else {
+				AssertFilesEqual(dh.t, inPath, outPath)
+			}
+
 			return nil
 		})
 	}

itests/kit/deals_state.go

@@ -14,7 +14,7 @@ func CategorizeDealState(dealStatus string) TestDealState {
 	switch dealStatus {
 	case "StorageDealFailing", "StorageDealError":
 		return TestDealStateFailed
-	case "StorageDealStaged", "StorageDealAwaitingPreCommit", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed":
+	case "StorageDealAwaitingPreCommit", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed":
 		return TestDealStateComplete
 	}
 	return TestDealStateInProgress