Merge branch 'master' into feat/cid-to-piece-idx

Merge pull request #8205 from s1mple1122/master
avoid panic
2022-02-28 12:57:11 -07:00 · 2022-02-28 13:31:33 +00:00 · 2022-02-28 13:31:25 +00:00 · 2022-02-28 16:51:15 +08:00 · 2022-02-28 12:40:43 +05:30 · 2022-02-27 19:25:35 -05:00
271 changed files with 16227 additions and 2039 deletions
@@ -44,13 +44,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -785,6 +785,11 @@ workflows:
          suite: itest-deals_512mb
          target: "./itests/deals_512mb_test.go"
      
+      - test:
+          name: test-itest-deals_anycid
+          suite: itest-deals_anycid
+          target: "./itests/deals_anycid_test.go"
+      
      - test:
          name: test-itest-deals_concurrent
          suite: itest-deals_concurrent
@@ -850,6 +855,11 @@ workflows:
          suite: itest-get_messages_in_ts
          target: "./itests/get_messages_in_ts_test.go"
      
+      - test:
+          name: test-itest-mempool
+          suite: itest-mempool
+          target: "./itests/mempool_test.go"
+      
      - test:
          name: test-itest-multisig
          suite: itest-multisig
@@ -44,13 +44,13 @@ commands:
      - restore_cache:
          name: Restore parameters cache
          keys:
-            - 'v25-2k-lotus-params'
+            - 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
      - run:  ./lotus fetch-params 2048
      - save_cache:
          name: Save parameters cache
-          key: 'v25-2k-lotus-params'
+          key: 'v26-2k-lotus-params'
          paths:
            - /var/tmp/filecoin-proof-parameters/
  install_ipfs:
@@ -12,10 +12,10 @@

 Before you mark the PR ready for review, please make sure that:
 - [ ] All commits have a clear commit message.
- [ ] The PR title is in the form of `<PR type>: <area>: <change being made>`
+- [ ] The PR title is in the form of of `<PR type>: <area>: <change being made>`
    - example: ` fix: mempool: Introduce a cache for valid signatures`
-    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_, _misc_,_perf_, _refactor_, _revert_, _style_, _test_
-    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_
+    - `PR type`: _fix_, _feat_, _INTERFACE BREAKING CHANGE_, _CONSENSUS BREAKING_, _build_, _chore_, _ci_, _docs_,_perf_, _refactor_, _revert_, _style_, _test_
+    - `area`: _api_, _chain_, _state_, _vm_, _data transfer_, _market_, _mempool_, _message_, _block production_, _multisig_, _networking_, _paychan_, _proving_, _sealing_, _wallet_, _deps_
 - [ ] This PR has tests for new functionality or change in behaviour
 - [ ] If new user-facing features are introduced, clear usage guidelines and / or documentation updates should be included in https://lotus.filecoin.io or [Discussion Tutorials.](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
 - [ ] CI is green
@@ -1,36 +1,57 @@
 # Lotus changelog

-# 1.14.0-rc3 / 2022-01-12
+# 1.14.2 / 2022-02-24

-This is the third release candidate for the mandatory release v1.14.0 of Lotus that introduces [Filecoin network v15, 
+This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 
+
+Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
+
+Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out! 
+
+## Bug Fixes
+- fix lotus-bench for sealing jobs (#8173)
+- fix:sealing:really-do-it flag for abort upgrade (#8181)
+- fix:proving:post check sector handles snap deals replica faults (#8177)
+- fix: sealing: missing file type (#8180)
+
+## Others
+- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong 
+  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore, 
+  we want to retract it and users may use v1.14.1&^.
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @zenground0 | 2 | +73/-58 | 12 |
+| @eben.xie | 1 | +7/-0 | 1 |
+| @jennijuju | 1 | +4/-0 | 1 |
+| @jennijuju | 1 | +2/-1 | 1 |
+| @ribasushi | 1 | +2/-0 | 1 |
+
+# 1.14.1 / 2022-02-18
+
+This is an **optional** release of lotus, that fixes the incorrect *comment* of network v15 OhSnap upgrade **date**. Note the actual upgrade epoch in [v1.14.0](https://github.com/filecoin-project/lotus/releases/tag/v1.14.0) was correct.
+
+# 1.14.0 / 2022-02-17
+
+This is a MANDATORY release of Lotus that introduces [Filecoin network v15, 
 codenamed the OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).

-The OhSnap upgrade introduces the following FIPs, delivered in [actors v7-rc1](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0-rc1):
+The network is scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, and worker(s).
+
+The OhSnap upgrade introduces the following FIPs, delivered in [actors v7](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0):
 - [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
 - [FIP-0028 Remove Datacap from Verified clients](https://github.com/filecoin-project/FIPs/pull/226)

-Note: 
- This release is built on top of lotus v1.13.2. Enterprising users like storage providers, data brokers and others 
-  are recommended to test out v1.15.0-rc1(coming on Jan 18th, 2022) for latest new features, improvements and bug 
-  fixes and be ready for the upgrade!
- This release candidate uses temporary proof params for Snap Deals.
- It only sets upgrade epoch for Butterfly-SnapNet, and it does not set the upgrade epochs for calibration and mainnet.
+It is recommended that storage providers download the new params before updating their node, miner, and workers. To do so:

-## Butterfly - SnapNet
+- Download Lotus v1.14.0 or later
+- run `make lotus-shed`
+- run `./lotus-shed fetch-params` with the appropriate `proving-params` flag
+- Upgrade the Lotus daemon and miner **when the previous step is complete**

-The ButterFly-SnapNet will be upgraded to Network v15 OhSnap at epoch 30262, around 2022-01-17T19:00:00Z. 
-
-To join the network, simply build lotus by running `make butterflynet`. 
-
-The network supports three sector sizes, 512MiB, 32GiB and 64GiB. Temporary proof params for Snap Deals should be downloaded upon your node restarts. 
- - The parameters are pinged on IPFS gateway https://proofs.filecoin.io/ipfs/ and the CIDs can be found [here](https://github.com/filecoin-project/lotus/blob/edd3486d2cf53b960382e9cda6671e647844aa41/build/proof-params/parameters.json), please let the lotus team know in #lotus-ohsnap if the params are not fetched automatically. You can also download the params manually from s3://proof-params-ap/filecoin-snapdeal-parameters/.
-
-*SnapNet Resources*:
- [Faucet](https://faucet.butterfly.fildev.network/)
- [Stats Dashboard](https://stats.butterfly.fildev.network/d/z6FtI92Zz/chain?orgId=1&refresh=25s&from=now-30m&to=now&kiosk)
- For questions and feedbacks:
-  - Primary: [Discussion](https://github.com/filecoin-project/lotus/discussions/7935)
-  - Secondary: #lotus-ohsnap in Filecoin Slack
+All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
  
 ## New Features and Changes
 - Integrate actor v7-rc1:
@@ -65,24 +86,20 @@ The network supports three sector sizes, 512MiB, 32GiB and 64GiB. Temporary proo

 | Contributor | Commits | Lines ± | Files Changed |
 |-------------|---------|---------|---------------|
-| Aayush Rajasekaran | 56 | +5866/-2523 | 316 |
+| Aayush Rajasekaran | 41 | +5538/-1205 | 189 |
 | zenground0 | 11 | +3316/-524 | 124 |
-| vyzo | 38 | +722/-485 | 117 |
-| Rod Vagg | 2 | +6/-941 | 4 |
-| whyrusleeping | 2 | +376/-339 | 27 |
+| Jennifer Wang | 29 | +714/-599 | 68 |
 | ZenGround0 | 3 | +263/-25 | 11 |
 | c r | 2 | +198/-30 | 6 |
-| Łukasz Magiera | 2 | +184/-3 | 3 |
-| Jennifer Wang | 14 | +97/-69 | 41 |
-| Whyrusleeping | 1 | +76/-70 | 8 |
+| vyzo | 4 | +189/-7 | 7 |
+| Aayush | 11 | +146/-48 | 49 |
 | web3-bot | 10 | +99/-17 | 10 |
 | Steven Allen | 1 | +55/-37 | 1 |
 | Jiaying Wang | 5 | +30/-8 | 5 |
-| Hannah Howard | 1 | +4/-29 | 3 |
-| dirkmc | 1 | +11/-0 | 1 |
 | Jakub Sztandera | 2 | +8/-3 | 3 |
-| Colin Kennedy | 1 | +4/-2 | 1 |
-
+| Łukasz Magiera | 1 | +3/-3 | 2 |
+| Travis Person | 1 | +2/-2 | 2 |
+| Rod Vagg | 1 | +2/-2 | 2 |

 # v1.13.2 /  2022-01-09

@@ -345,6 +345,8 @@ gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci
 	@echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli"
 .PHONY: gen

+jen: gen
+
 snap: lotus lotus-miner lotus-worker
 	snapcraft
 	# snapcraft upload ./lotus_*.snap
@@ -900,6 +900,7 @@ type QueryOffer struct {
 	Size                    uint64
 	MinPrice                types.BigInt
 	UnsealPrice             types.BigInt
+	PricePerByte            abi.TokenAmount
 	PaymentInterval         uint64
 	PaymentIntervalIncrease uint64
 	Miner                   address.Address
@@ -45,8 +45,9 @@ type Gateway interface {
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
@@ -51,6 +51,15 @@ type Net interface {
 	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
 	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read

+	NetProtectAdd(ctx context.Context, acl []peer.ID) error    //perm:admin
+	NetProtectRemove(ctx context.Context, acl []peer.ID) error //perm:admin
+	NetProtectList(ctx context.Context) ([]peer.ID, error)     //perm:read
+
+	// ResourceManager API
+	NetStat(ctx context.Context, scope string) (NetStat, error)          //perm:read
+	NetLimit(ctx context.Context, scope string) (NetLimit, error)        //perm:read
+	NetSetLimit(ctx context.Context, scope string, limit NetLimit) error //perm:admin
+
 	// ID returns peerID of libp2p node backing this API
 	ID(context.Context) (peer.ID, error) //perm:read
 }
@@ -132,6 +132,7 @@ type StorageMiner interface {
 	ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true
 	ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error           //perm:admin retry:true
 	ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                       //perm:admin retry:true
+	ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                           //perm:admin retry:true
 	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                 //perm:admin retry:true
 	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
 	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                                     //perm:admin retry:true
@@ -153,6 +154,7 @@ type StorageMiner interface {
 	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
 	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
 	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)                                                                                               //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                 //perm:admin

 	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
@@ -168,6 +170,8 @@ type StorageMiner interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
 	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
 	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
+	// MarketDataTransferDiagnostics generates debugging information about current data transfers over graphsync
+	MarketDataTransferDiagnostics(ctx context.Context, p peer.ID) (*TransferDiagnostics, error) //perm:write
 	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
 	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
@@ -218,6 +222,16 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin

+	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
+	// so they can download its index
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// IndexerAnnounceAllDeals informs the indexer nodes aboutall active deals.
+	IndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid cid.Cid) ([]DagstoreShardInfo, error) //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (MinerSubsystems, error) //perm:read
@@ -252,7 +266,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin

-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin

 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }
@@ -39,6 +39,7 @@ type Worker interface {
 	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
 	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
 	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
+	FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                         //perm:admin
 	ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)                                                                                       //perm:admin
 	ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error)                                                              //perm:admin
 	ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)                  //perm:admin
@@ -1,6 +1,7 @@
 package docgen

 import (
+	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
@@ -15,6 +16,7 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"
 	"github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -120,6 +122,7 @@ func init() {
 	addExample(api.FullAPIVersion1)
 	addExample(api.PCHInbound)
 	addExample(time.Minute)
+	addExample(graphsync.RequestID(4))
 	addExample(datatransfer.TransferID(3))
 	addExample(datatransfer.Ongoing)
 	addExample(storeIDExample)
@@ -250,10 +253,18 @@ func init() {
 	addExample(map[abi.SectorNumber]string{
 		123: "can't acquire read lock",
 	})
+	addExample(json.RawMessage(`"json raw message"`))
 	addExample(map[api.SectorState]int{
 		api.SectorState(sealing.Proving): 120,
 	})
 	addExample([]abi.SectorNumber{123, 124})
+	addExample([]storiface.SectorLock{
+		{
+			Sector: abi.SectorID{Number: 123, Miner: 1000},
+			Write:  [storiface.FileTypes]uint{0, 0, 1},
+			Read:   [storiface.FileTypes]uint{2, 3, 0},
+		},
+	})

 	// worker specific
 	addExample(storiface.AcquireMove)
@@ -289,6 +300,34 @@ func init() {
 		Error: "<error>",
 	})
 	addExample(storiface.ResourceTable)
+	addExample(network.ScopeStat{
+		Memory:             123,
+		NumStreamsInbound:  1,
+		NumStreamsOutbound: 2,
+		NumConnsInbound:    3,
+		NumConnsOutbound:   4,
+		NumFD:              5,
+	})
+	addExample(map[string]network.ScopeStat{
+		"abc": {
+			Memory:             123,
+			NumStreamsInbound:  1,
+			NumStreamsOutbound: 2,
+			NumConnsInbound:    3,
+			NumConnsOutbound:   4,
+			NumFD:              5,
+		}})
+	addExample(api.NetLimit{
+		Memory:          123,
+		StreamsInbound:  1,
+		StreamsOutbound: 2,
+		Streams:         3,
+		ConnsInbound:    3,
+		ConnsOutbound:   4,
+		Conns:           4,
+		FD:              5,
+	})
+
 }

 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {
@@ -339,7 +378,7 @@ func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
+		out = reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
 		return out.Interface()
 	case reflect.Chan:
 		return ExampleValue(method, t.Elem(), nil)
@@ -1811,6 +1811,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }

+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1841,6 +1856,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1856,6 +1914,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }

+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // NodeStatus mocks base method.
 func (m *MockFullNode) NodeStatus(arg0 context.Context, arg1 bool) (api.NodeStatus, error) {
 	m.ctrl.T.Helper()
@@ -516,6 +516,8 @@ type GatewayStruct struct {

 		MsigGetVested func(p0 context.Context, p1 address.Address, p2 types.TipSetKey, p3 types.TipSetKey) (types.BigInt, error) ``

+		MsigGetVestingSchedule func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) ``
+
 		StateAccountKey func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) ``

 		StateDealProviderCollateralBounds func(p0 context.Context, p1 abi.PaddedPieceSize, p2 bool, p3 types.TipSetKey) (DealCollateralBounds, error) ``
@@ -587,11 +589,23 @@ type NetStruct struct {

 		NetFindPeer func(p0 context.Context, p1 peer.ID) (peer.AddrInfo, error) `perm:"read"`

+		NetLimit func(p0 context.Context, p1 string) (NetLimit, error) `perm:"read"`
+
 		NetPeerInfo func(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) `perm:"read"`

 		NetPeers func(p0 context.Context) ([]peer.AddrInfo, error) `perm:"read"`

+		NetProtectAdd func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
+		NetProtectList func(p0 context.Context) ([]peer.ID, error) `perm:"read"`
+
+		NetProtectRemove func(p0 context.Context, p1 []peer.ID) error `perm:"admin"`
+
 		NetPubsubScores func(p0 context.Context) ([]PubsubScore, error) `perm:"read"`
+
+		NetSetLimit func(p0 context.Context, p1 string, p2 NetLimit) error `perm:"admin"`
+
+		NetStat func(p0 context.Context, p1 string) (NetStat, error) `perm:"read"`
 	}
 }

@@ -619,7 +633,7 @@ type StorageMinerStruct struct {

 		ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"`

-		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
+		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`

 		ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`

@@ -633,6 +647,8 @@ type StorageMinerStruct struct {

 		DagstoreListShards func(p0 context.Context) ([]DagstoreShardInfo, error) `perm:"read"`

+		DagstoreLookupPieces func(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) `perm:"admin"`
+
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`

 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -667,8 +683,14 @@ type StorageMinerStruct struct {

 		DealsSetPieceCidBlocklist func(p0 context.Context, p1 []cid.Cid) error `perm:"admin"`

+		IndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"`
+
+		IndexerAnnounceDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"`
+
 		MarketCancelDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`

+		MarketDataTransferDiagnostics func(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) `perm:"write"`
+
 		MarketDataTransferUpdates func(p0 context.Context) (<-chan DataTransferChannel, error) `perm:"write"`

 		MarketGetAsk func(p0 context.Context) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
@@ -715,6 +737,8 @@ type StorageMinerStruct struct {

 		ReturnFetch func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`

+		ReturnFinalizeReplicaUpdate func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
+
 		ReturnFinalizeSector func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`

 		ReturnGenerateSectorKeyFromData func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"`
@@ -807,6 +831,8 @@ type StorageMinerStruct struct {

 		StorageFindSector func(p0 context.Context, p1 abi.SectorID, p2 storiface.SectorFileType, p3 abi.SectorSize, p4 bool) ([]stores.SectorStorageInfo, error) `perm:"admin"`

+		StorageGetLocks func(p0 context.Context) (storiface.SectorLocks, error) `perm:"admin"`
+
 		StorageInfo func(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) `perm:"admin"`

 		StorageList func(p0 context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"`
@@ -864,6 +890,8 @@ type WorkerStruct struct {

 		Fetch func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType, p3 storiface.PathType, p4 storiface.AcquireMode) (storiface.CallID, error) `perm:"admin"`

+		FinalizeReplicaUpdate func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`
+
 		FinalizeSector func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"`

 		GenerateSectorKeyFromData func(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) `perm:"admin"`
@@ -3271,6 +3299,17 @@ func (s *GatewayStub) MsigGetVested(p0 context.Context, p1 address.Address, p2 t
 	return *new(types.BigInt), ErrNotSupported
 }

+func (s *GatewayStruct) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	if s.Internal.MsigGetVestingSchedule == nil {
+		return *new(MsigVesting), ErrNotSupported
+	}
+	return s.Internal.MsigGetVestingSchedule(p0, p1, p2)
+}
+
+func (s *GatewayStub) MsigGetVestingSchedule(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MsigVesting, error) {
+	return *new(MsigVesting), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateAccountKey(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (address.Address, error) {
 	if s.Internal.StateAccountKey == nil {
 		return *new(address.Address), ErrNotSupported
@@ -3623,6 +3662,17 @@ func (s *NetStub) NetFindPeer(p0 context.Context, p1 peer.ID) (peer.AddrInfo, er
 	return *new(peer.AddrInfo), ErrNotSupported
 }

+func (s *NetStruct) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
+	if s.Internal.NetLimit == nil {
+		return *new(NetLimit), ErrNotSupported
+	}
+	return s.Internal.NetLimit(p0, p1)
+}
+
+func (s *NetStub) NetLimit(p0 context.Context, p1 string) (NetLimit, error) {
+	return *new(NetLimit), ErrNotSupported
+}
+
 func (s *NetStruct) NetPeerInfo(p0 context.Context, p1 peer.ID) (*ExtendedPeerInfo, error) {
 	if s.Internal.NetPeerInfo == nil {
 		return nil, ErrNotSupported
@@ -3645,6 +3695,39 @@ func (s *NetStub) NetPeers(p0 context.Context) ([]peer.AddrInfo, error) {
 	return *new([]peer.AddrInfo), ErrNotSupported
 }

+func (s *NetStruct) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectAdd == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectAdd(p0, p1)
+}
+
+func (s *NetStub) NetProtectAdd(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	if s.Internal.NetProtectList == nil {
+		return *new([]peer.ID), ErrNotSupported
+	}
+	return s.Internal.NetProtectList(p0)
+}
+
+func (s *NetStub) NetProtectList(p0 context.Context) ([]peer.ID, error) {
+	return *new([]peer.ID), ErrNotSupported
+}
+
+func (s *NetStruct) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	if s.Internal.NetProtectRemove == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetProtectRemove(p0, p1)
+}
+
+func (s *NetStub) NetProtectRemove(p0 context.Context, p1 []peer.ID) error {
+	return ErrNotSupported
+}
+
 func (s *NetStruct) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	if s.Internal.NetPubsubScores == nil {
 		return *new([]PubsubScore), ErrNotSupported
@@ -3656,6 +3739,28 @@ func (s *NetStub) NetPubsubScores(p0 context.Context) ([]PubsubScore, error) {
 	return *new([]PubsubScore), ErrNotSupported
 }

+func (s *NetStruct) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
+	if s.Internal.NetSetLimit == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.NetSetLimit(p0, p1, p2)
+}
+
+func (s *NetStub) NetSetLimit(p0 context.Context, p1 string, p2 NetLimit) error {
+	return ErrNotSupported
+}
+
+func (s *NetStruct) NetStat(p0 context.Context, p1 string) (NetStat, error) {
+	if s.Internal.NetStat == nil {
+		return *new(NetStat), ErrNotSupported
+	}
+	return s.Internal.NetStat(p0, p1)
+}
+
+func (s *NetStub) NetStat(p0 context.Context, p1 string) (NetStat, error) {
+	return *new(NetStat), ErrNotSupported
+}
+
 func (s *SignableStruct) Sign(p0 context.Context, p1 SignFunc) error {
 	if s.Internal.Sign == nil {
 		return ErrNotSupported
@@ -3700,14 +3805,14 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres
 	return *new(abi.SectorSize), ErrNotSupported
 }

-func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	if s.Internal.CheckProvable == nil {
 		return *new(map[abi.SectorNumber]string), ErrNotSupported
 	}
-	return s.Internal.CheckProvable(p0, p1, p2, p3)
+	return s.Internal.CheckProvable(p0, p1, p2, p3, p4)
 }

-func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	return *new(map[abi.SectorNumber]string), ErrNotSupported
 }

@@ -3777,6 +3882,17 @@ func (s *StorageMinerStub) DagstoreListShards(p0 context.Context) ([]DagstoreSha
 	return *new([]DagstoreShardInfo), ErrNotSupported
 }

+func (s *StorageMinerStruct) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	if s.Internal.DagstoreLookupPieces == nil {
+		return *new([]DagstoreShardInfo), ErrNotSupported
+	}
+	return s.Internal.DagstoreLookupPieces(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreLookupPieces(p0 context.Context, p1 cid.Cid) ([]DagstoreShardInfo, error) {
+	return *new([]DagstoreShardInfo), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DagstoreRecoverShard(p0 context.Context, p1 string) error {
 	if s.Internal.DagstoreRecoverShard == nil {
 		return ErrNotSupported
@@ -3964,6 +4080,28 @@ func (s *StorageMinerStub) DealsSetPieceCidBlocklist(p0 context.Context, p1 []ci
 	return ErrNotSupported
 }

+func (s *StorageMinerStruct) IndexerAnnounceAllDeals(p0 context.Context) error {
+	if s.Internal.IndexerAnnounceAllDeals == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceAllDeals(p0)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceAllDeals(p0 context.Context) error {
+	return ErrNotSupported
+}
+
+func (s *StorageMinerStruct) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	if s.Internal.IndexerAnnounceDeal == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.IndexerAnnounceDeal(p0, p1)
+}
+
+func (s *StorageMinerStub) IndexerAnnounceDeal(p0 context.Context, p1 cid.Cid) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketCancelDataTransfer(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error {
 	if s.Internal.MarketCancelDataTransfer == nil {
 		return ErrNotSupported
@@ -3975,6 +4113,17 @@ func (s *StorageMinerStub) MarketCancelDataTransfer(p0 context.Context, p1 datat
 	return ErrNotSupported
 }

+func (s *StorageMinerStruct) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
+	if s.Internal.MarketDataTransferDiagnostics == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.MarketDataTransferDiagnostics(p0, p1)
+}
+
+func (s *StorageMinerStub) MarketDataTransferDiagnostics(p0 context.Context, p1 peer.ID) (*TransferDiagnostics, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *StorageMinerStruct) MarketDataTransferUpdates(p0 context.Context) (<-chan DataTransferChannel, error) {
 	if s.Internal.MarketDataTransferUpdates == nil {
 		return nil, ErrNotSupported
@@ -4228,6 +4377,17 @@ func (s *StorageMinerStub) ReturnFetch(p0 context.Context, p1 storiface.CallID,
 	return ErrNotSupported
 }

+func (s *StorageMinerStruct) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
+	if s.Internal.ReturnFinalizeReplicaUpdate == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.ReturnFinalizeReplicaUpdate(p0, p1, p2)
+}
+
+func (s *StorageMinerStub) ReturnFinalizeReplicaUpdate(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) ReturnFinalizeSector(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error {
 	if s.Internal.ReturnFinalizeSector == nil {
 		return ErrNotSupported
@@ -4734,6 +4894,17 @@ func (s *StorageMinerStub) StorageFindSector(p0 context.Context, p1 abi.SectorID
 	return *new([]stores.SectorStorageInfo), ErrNotSupported
 }

+func (s *StorageMinerStruct) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
+	if s.Internal.StorageGetLocks == nil {
+		return *new(storiface.SectorLocks), ErrNotSupported
+	}
+	return s.Internal.StorageGetLocks(p0)
+}
+
+func (s *StorageMinerStub) StorageGetLocks(p0 context.Context) (storiface.SectorLocks, error) {
+	return *new(storiface.SectorLocks), ErrNotSupported
+}
+
 func (s *StorageMinerStruct) StorageInfo(p0 context.Context, p1 stores.ID) (stores.StorageInfo, error) {
 	if s.Internal.StorageInfo == nil {
 		return *new(stores.StorageInfo), ErrNotSupported
@@ -4954,6 +5125,17 @@ func (s *WorkerStub) Fetch(p0 context.Context, p1 storage.SectorRef, p2 storifac
 	return *new(storiface.CallID), ErrNotSupported
 }

+func (s *WorkerStruct) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
+	if s.Internal.FinalizeReplicaUpdate == nil {
+		return *new(storiface.CallID), ErrNotSupported
+	}
+	return s.Internal.FinalizeReplicaUpdate(p0, p1, p2)
+}
+
+func (s *WorkerStub) FinalizeReplicaUpdate(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
+	return *new(storiface.CallID), ErrNotSupported
+}
+
 func (s *WorkerStruct) FinalizeSector(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) {
 	if s.Internal.FinalizeSector == nil {
 		return *new(storiface.CallID), ErrNotSupported
@@ -10,7 +10,9 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-graphsync"

+	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	ma "github.com/multiformats/go-multiaddr"
@@ -53,6 +55,30 @@ type MessageSendSpec struct {
 	MaxFee abi.TokenAmount
 }

+// GraphSyncDataTransfer provides diagnostics on a data transfer happening over graphsync
+type GraphSyncDataTransfer struct {
+	// GraphSync request id for this transfer
+	RequestID graphsync.RequestID
+	// Graphsync state for this transfer
+	RequestState string
+	// If a channel ID is present, indicates whether this is the current graphsync request for this channel
+	// (could have changed in a restart)
+	IsCurrentChannelRequest bool
+	// Data transfer channel ID for this transfer
+	ChannelID *datatransfer.ChannelID
+	// Data transfer state for this transfer
+	ChannelState *DataTransferChannel
+	// Diagnostic information about this request -- and unexpected inconsistencies in
+	// request state
+	Diagnostics []string
+}
+
+// TransferDiagnostics give current information about transfers going over graphsync that may be helpful for debugging
+type TransferDiagnostics struct {
+	ReceivingTransfers []*GraphSyncDataTransfer
+	SendingTransfers   []*GraphSyncDataTransfer
+}
+
 type DataTransferChannel struct {
 	TransferID  datatransfer.TransferID
 	Status      datatransfer.Status
@@ -104,6 +130,28 @@ type NetBlockList struct {
 	IPSubnets []string
 }

+type NetStat struct {
+	System    *network.ScopeStat           `json:",omitempty"`
+	Transient *network.ScopeStat           `json:",omitempty"`
+	Services  map[string]network.ScopeStat `json:",omitempty"`
+	Protocols map[string]network.ScopeStat `json:",omitempty"`
+	Peers     map[string]network.ScopeStat `json:",omitempty"`
+}
+
+type NetLimit struct {
+	Dynamic bool `json:",omitempty"`
+	// set if Dynamic is false
+	Memory int64 `json:",omitempty"`
+	// set if Dynamic is true
+	MemoryFraction float64 `json:",omitempty"`
+	MinMemory      int64   `json:",omitempty"`
+	MaxMemory      int64   `json:",omitempty"`
+
+	Streams, StreamsInbound, StreamsOutbound int
+	Conns, ConnsInbound, ConnsOutbound       int
+	FD                                       int
+}
+
 type ExtendedPeerInfo struct {
 	ID          peer.ID
 	Agent       string
@@ -1724,6 +1724,21 @@ func (mr *MockFullNodeMockRecorder) NetFindPeer(arg0, arg1 interface{}) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetFindPeer", reflect.TypeOf((*MockFullNode)(nil).NetFindPeer), arg0, arg1)
 }

+// NetLimit mocks base method.
+func (m *MockFullNode) NetLimit(arg0 context.Context, arg1 string) (api.NetLimit, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetLimit", arg0, arg1)
+	ret0, _ := ret[0].(api.NetLimit)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetLimit indicates an expected call of NetLimit.
+func (mr *MockFullNodeMockRecorder) NetLimit(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetLimit", reflect.TypeOf((*MockFullNode)(nil).NetLimit), arg0, arg1)
+}
+
 // NetPeerInfo mocks base method.
 func (m *MockFullNode) NetPeerInfo(arg0 context.Context, arg1 peer.ID) (*api.ExtendedPeerInfo, error) {
 	m.ctrl.T.Helper()
@@ -1754,6 +1769,49 @@ func (mr *MockFullNodeMockRecorder) NetPeers(arg0 interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPeers", reflect.TypeOf((*MockFullNode)(nil).NetPeers), arg0)
 }

+// NetProtectAdd mocks base method.
+func (m *MockFullNode) NetProtectAdd(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectAdd", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectAdd indicates an expected call of NetProtectAdd.
+func (mr *MockFullNodeMockRecorder) NetProtectAdd(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectAdd", reflect.TypeOf((*MockFullNode)(nil).NetProtectAdd), arg0, arg1)
+}
+
+// NetProtectList mocks base method.
+func (m *MockFullNode) NetProtectList(arg0 context.Context) ([]peer.ID, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectList", arg0)
+	ret0, _ := ret[0].([]peer.ID)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetProtectList indicates an expected call of NetProtectList.
+func (mr *MockFullNodeMockRecorder) NetProtectList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectList", reflect.TypeOf((*MockFullNode)(nil).NetProtectList), arg0)
+}
+
+// NetProtectRemove mocks base method.
+func (m *MockFullNode) NetProtectRemove(arg0 context.Context, arg1 []peer.ID) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetProtectRemove", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetProtectRemove indicates an expected call of NetProtectRemove.
+func (mr *MockFullNodeMockRecorder) NetProtectRemove(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetProtectRemove", reflect.TypeOf((*MockFullNode)(nil).NetProtectRemove), arg0, arg1)
+}
+
 // NetPubsubScores mocks base method.
 func (m *MockFullNode) NetPubsubScores(arg0 context.Context) ([]api.PubsubScore, error) {
 	m.ctrl.T.Helper()
@@ -1769,6 +1827,35 @@ func (mr *MockFullNodeMockRecorder) NetPubsubScores(arg0 interface{}) *gomock.Ca
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetPubsubScores", reflect.TypeOf((*MockFullNode)(nil).NetPubsubScores), arg0)
 }

+// NetSetLimit mocks base method.
+func (m *MockFullNode) NetSetLimit(arg0 context.Context, arg1 string, arg2 api.NetLimit) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetSetLimit", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NetSetLimit indicates an expected call of NetSetLimit.
+func (mr *MockFullNodeMockRecorder) NetSetLimit(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetSetLimit", reflect.TypeOf((*MockFullNode)(nil).NetSetLimit), arg0, arg1, arg2)
+}
+
+// NetStat mocks base method.
+func (m *MockFullNode) NetStat(arg0 context.Context, arg1 string) (api.NetStat, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NetStat", arg0, arg1)
+	ret0, _ := ret[0].(api.NetStat)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NetStat indicates an expected call of NetStat.
+func (mr *MockFullNodeMockRecorder) NetStat(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetStat", reflect.TypeOf((*MockFullNode)(nil).NetStat), arg0, arg1)
+}
+
 // PaychAllocateLane mocks base method.
 func (m *MockFullNode) PaychAllocateLane(arg0 context.Context, arg1 address.Address) (uint64, error) {
 	m.ctrl.T.Helper()
@@ -57,7 +57,7 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)

-	MinerAPIVersion0  = newVer(1, 3, 0)
+	MinerAPIVersion0  = newVer(1, 4, 0)
 	WorkerAPIVersion0 = newVer(1, 5, 0)
 )

@@ -0,0 +1,21 @@
+package blockstore
+
+import (
+	"context"
+)
+
+type hotViewKey struct{}
+
+var hotView = hotViewKey{}
+
+// WithHotView constructs a new context with an option that provides a hint to the blockstore
+// (e.g. the splitstore) that the object (and its ipld references) should be kept hot.
+func WithHotView(ctx context.Context) context.Context {
+	return context.WithValue(ctx, hotView, struct{}{})
+}
+
+// IsHotView returns true if the hot view option is set in the context
+func IsHotView(ctx context.Context) bool {
+	v := ctx.Value(hotView)
+	return v != nil
+}
@@ -49,10 +49,11 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration:
  blockstore and discards writes; this is necessary to support syncing from a snapshot.
 - `MarkSetType` -- specifies the type of markset to use during compaction.
  The markset is the data structure used by compaction/gc to track live objects.
-  The default value is `"map"`, which will use an in-memory map; if you are limited
-  in memory (or indeed see compaction run out of memory), you can also specify
-  `"badger"` which will use an disk backed markset, using badger. This will use
-  much less memory, but will also make compaction slower.
+  The default value is "badger", which will use a disk backed markset using badger.
+  If you have a lot of memory (48G or more) you can also use "map", which will use
+  an in memory markset, speeding up compaction at the cost of higher memory usage.
+  Note: If you are using a VPS with a network volume, you need to provision at least
+  3000 IOPs with the badger markset.
 - `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4
  finalities maintained by default, to maintain messages and message receipts in the
  hotstore. This is useful for assistive nodes that want to support syncing for other
@@ -105,6 +106,12 @@ Compaction works transactionally with the following algorithm:
  - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live
 - We then end the transaction and compact/gc the hotstore.

+As of [#8008](https://github.com/filecoin-project/lotus/pull/8008) the compaction algorithm has been
+modified to eliminate sorting and maintain the cold object set on disk. This drastically reduces
+memory usage; in fact, when using badger as the markset compaction uses very little memory, and
+it should be now possible to run splitstore with 32GB of RAM or less without danger of running out of
+memory during compaction.
+
 ## Garbage Collection

 TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577)
@@ -0,0 +1,118 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+type Checkpoint struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+func NewCheckpoint(path string) (*Checkpoint, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &Checkpoint{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func OpenCheckpoint(path string) (*Checkpoint, cid.Cid, error) {
+	filein, err := os.Open(path)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for reading: %w", err)
+	}
+	defer filein.Close() //nolint:errcheck
+
+	bufin := bufio.NewReader(filein)
+	start, err := readRawCid(bufin, nil)
+	if err != nil && err != io.EOF {
+		return nil, cid.Undef, xerrors.Errorf("error reading cid from checkpoint: %w", err)
+	}
+
+	fileout, err := os.OpenFile(path, os.O_WRONLY|os.O_SYNC, 0644)
+	if err != nil {
+		return nil, cid.Undef, xerrors.Errorf("error opening checkpoint for writing: %w", err)
+	}
+	bufout := bufio.NewWriter(fileout)
+
+	return &Checkpoint{
+		file: fileout,
+		buf:  bufout,
+	}, start, nil
+}
+
+func (cp *Checkpoint) Set(c cid.Cid) error {
+	if _, err := cp.file.Seek(0, io.SeekStart); err != nil {
+		return xerrors.Errorf("error seeking beginning of checkpoint: %w", err)
+	}
+
+	if err := writeRawCid(cp.buf, c, true); err != nil {
+		return xerrors.Errorf("error writing cid to checkpoint: %w", err)
+	}
+
+	return nil
+}
+
+func (cp *Checkpoint) Close() error {
+	if cp.file == nil {
+		return nil
+	}
+
+	err := cp.file.Close()
+	cp.file = nil
+	cp.buf = nil
+
+	return err
+}
+
+func readRawCid(buf *bufio.Reader, hbuf []byte) (cid.Cid, error) {
+	sz, err := buf.ReadByte()
+	if err != nil {
+		return cid.Undef, err // don't wrap EOF as it is not an error here
+	}
+
+	if hbuf == nil {
+		hbuf = make([]byte, int(sz))
+	} else {
+		hbuf = hbuf[:int(sz)]
+	}
+
+	if _, err := io.ReadFull(buf, hbuf); err != nil {
+		return cid.Undef, xerrors.Errorf("error reading hash: %w", err) // wrap EOF, it's corrupt
+	}
+
+	hash, err := mh.Cast(hbuf)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error casting multihash: %w", err)
+	}
+
+	return cid.NewCidV1(cid.Raw, hash), nil
+}
+
+func writeRawCid(buf *bufio.Writer, c cid.Cid, flush bool) error {
+	hash := c.Hash()
+	if err := buf.WriteByte(byte(len(hash))); err != nil {
+		return err
+	}
+	if _, err := buf.Write(hash); err != nil {
+		return err
+	}
+	if flush {
+		return buf.Flush()
+	}
+
+	return nil
+}
@@ -0,0 +1,147 @@
+package splitstore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestCheckpoint(t *testing.T) {
+	dir, err := ioutil.TempDir("", "checkpoint.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	path := filepath.Join(dir, "checkpoint")
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	cp, err := NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err := OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// also test correct operation with an empty checkpoint
+	cp, err = NewCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if start.Defined() {
+		t.Fatal("expected start to be undefined")
+	}
+
+	if err := cp.Set(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k2) {
+		t.Fatalf("expected start to be %s; got %s", k2, start)
+	}
+
+	if err := cp.Set(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := cp.Set(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cp, start, err = OpenCheckpoint(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !start.Equals(k4) {
+		t.Fatalf("expected start to be %s; got %s", k4, start)
+	}
+
+	if err := cp.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -0,0 +1,102 @@
+package splitstore
+
+import (
+	"bufio"
+	"io"
+	"os"
+
+	"golang.org/x/xerrors"
+
+	cid "github.com/ipfs/go-cid"
+)
+
+type ColdSetWriter struct {
+	file *os.File
+	buf  *bufio.Writer
+}
+
+type ColdSetReader struct {
+	file *os.File
+	buf  *bufio.Reader
+}
+
+func NewColdSetWriter(path string) (*ColdSetWriter, error) {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating coldset: %w", err)
+	}
+	buf := bufio.NewWriter(file)
+
+	return &ColdSetWriter{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func NewColdSetReader(path string) (*ColdSetReader, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening coldset: %w", err)
+	}
+	buf := bufio.NewReader(file)
+
+	return &ColdSetReader{
+		file: file,
+		buf:  buf,
+	}, nil
+}
+
+func (s *ColdSetWriter) Write(c cid.Cid) error {
+	return writeRawCid(s.buf, c, false)
+}
+
+func (s *ColdSetWriter) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err1 := s.buf.Flush()
+	err2 := s.file.Close()
+	s.buf = nil
+	s.file = nil
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (s *ColdSetReader) ForEach(f func(cid.Cid) error) error {
+	hbuf := make([]byte, 256)
+	for {
+		next, err := readRawCid(s.buf, hbuf)
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return xerrors.Errorf("error reading coldset: %w", err)
+		}
+
+		if err := f(next); err != nil {
+			return err
+		}
+	}
+}
+
+func (s *ColdSetReader) Reset() error {
+	_, err := s.file.Seek(0, io.SeekStart)
+	return err
+}
+
+func (s *ColdSetReader) Close() error {
+	if s.file == nil {
+		return nil
+	}
+
+	err := s.file.Close()
+	s.file = nil
+	s.buf = nil
+
+	return err
+}
@@ -0,0 +1,99 @@
+package splitstore
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+func TestColdSet(t *testing.T) {
+	dir, err := ioutil.TempDir("", "coldset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	path := filepath.Join(dir, "coldset")
+
+	makeCid := func(i int) cid.Cid {
+		h, err := multihash.Sum([]byte(fmt.Sprintf("cid.%d", i)), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	const count = 1000
+	cids := make([]cid.Cid, 0, count)
+	for i := 0; i < count; i++ {
+		cids = append(cids, makeCid(i))
+	}
+
+	cw, err := NewColdSetWriter(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range cids {
+		if err := cw.Write(c); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := cw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	cr, err := NewColdSetReader(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	index := 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid %d; expected %s but got %s", index, cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cr.Reset(); err != nil {
+		t.Fatal(err)
+	}
+
+	index = 0
+	err = cr.ForEach(func(c cid.Cid) error {
+		if index >= count {
+			t.Fatal("too many cids")
+		}
+
+		if !c.Equals(cids[index]) {
+			t.Fatalf("wrong cid; expected %s but got %s", cids[index], c)
+		}
+
+		index++
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
@@ -10,39 +10,36 @@ import (

 var errMarkSetClosed = errors.New("markset closed")

-// MarkSet is a utility to keep track of seen CID, and later query for them.
-//
-// * If the expected dataset is large, it can be backed by a datastore (e.g. bbolt).
-// * If a probabilistic result is acceptable, it can be backed by a bloom filter
+// MarkSet is an interface for tracking CIDs during chain and object walks
 type MarkSet interface {
+	ObjectVisitor
 	Mark(cid.Cid) error
+	MarkMany([]cid.Cid) error
 	Has(cid.Cid) (bool, error)
 	Close() error
-	SetConcurrent()
-}

-type MarkSetVisitor interface {
-	MarkSet
-	ObjectVisitor
+	// BeginCriticalSection ensures that the markset is persisted to disk for recovery in case
+	// of abnormal termination during the critical section span.
+	BeginCriticalSection() error
+	// EndCriticalSection ends the critical section span.
+	EndCriticalSection()
 }

 type MarkSetEnv interface {
-	// Create creates a new markset within the environment.
-	// name is a unique name for this markset, mapped to the filesystem in disk-backed environments
+	// New creates a new markset within the environment.
+	// name is a unique name for this markset, mapped to the filesystem for on-disk persistence.
 	// sizeHint is a hint about the expected size of the markset
-	Create(name string, sizeHint int64) (MarkSet, error)
-	// CreateVisitor is like Create, but returns a wider interface that supports atomic visits.
-	// It may not be supported by some markset types (e.g. bloom).
-	CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error)
-	// SupportsVisitor returns true if the marksets created by this environment support the visitor interface.
-	SupportsVisitor() bool
+	New(name string, sizeHint int64) (MarkSet, error)
+	// Recover recovers an existing markset persisted on-disk.
+	Recover(name string) (MarkSet, error)
+	// Close closes the markset
 	Close() error
 }

 func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) {
 	switch mtype {
 	case "map":
-		return NewMapMarkSetEnv()
+		return NewMapMarkSetEnv(path)
 	case "badger":
 		return NewBadgerMarkSetEnv(path)
 	default:
@@ -3,6 +3,7 @@ package splitstore
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"

 	"golang.org/x/xerrors"
@@ -28,13 +29,13 @@ type BadgerMarkSet struct {
 	writers int
 	seqno   int
 	version int
+	persist bool

 	db   *badger.DB
 	path string
 }

 var _ MarkSet = (*BadgerMarkSet)(nil)
-var _ MarkSetVisitor = (*BadgerMarkSet)(nil)

 var badgerMarkSetBatchSize = 16384

@@ -48,11 +49,10 @@ func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) {
 	return &BadgerMarkSetEnv{path: msPath}, nil
 }

-func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet, error) {
-	name += ".tmp"
+func (e *BadgerMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
 	path := filepath.Join(e.path, name)

-	db, err := openTransientBadgerDB(path)
+	db, err := openBadgerDB(path, false)
 	if err != nil {
 		return nil, xerrors.Errorf("error creating badger db: %w", err)
 	}
@@ -68,18 +68,72 @@ func (e *BadgerMarkSetEnv) create(name string, sizeHint int64) (*BadgerMarkSet,
 	return ms, nil
 }

-func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *BadgerMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)

-func (e *BadgerMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	if _, err := os.Stat(path); err != nil {
+		return nil, xerrors.Errorf("error stating badger db path: %w", err)
+	}

-func (e *BadgerMarkSetEnv) SupportsVisitor() bool { return true }
+	db, err := openBadgerDB(path, true)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating badger db: %w", err)
+	}
+
+	ms := &BadgerMarkSet{
+		pend:    make(map[string]struct{}),
+		writing: make(map[int]map[string]struct{}),
+		db:      db,
+		path:    path,
+		persist: true,
+	}
+	ms.cond.L = &ms.mx
+
+	return ms, nil
+}

 func (e *BadgerMarkSetEnv) Close() error {
-	return os.RemoveAll(e.path)
+	return nil
+}
+
+func (s *BadgerMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+
+	if s.persist {
+		s.mx.Unlock()
+		return nil
+	}
+
+	var write bool
+	var seqno int
+	if len(s.pend) > 0 {
+		write = true
+		seqno = s.nextBatch()
+	}
+
+	s.persist = true
+	s.mx.Unlock()
+
+	if write {
+		// all writes sync once perist is true
+		return s.write(seqno)
+	}
+
+	// wait for any pending writes and sync
+	s.mx.Lock()
+	for s.writers > 0 {
+		s.cond.Wait()
+	}
+	s.mx.Unlock()
+
+	return s.db.Sync()
+}
+
+func (s *BadgerMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	s.persist = false
 }

 func (s *BadgerMarkSet) Mark(c cid.Cid) error {
@@ -99,6 +153,23 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error {
 	return nil
 }

+func (s *BadgerMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	if s.pend == nil {
+		s.mx.Unlock()
+		return errMarkSetClosed
+	}
+
+	write, seqno := s.putMany(batch)
+	s.mx.Unlock()
+
+	if write {
+		return s.write(seqno)
+	}
+
+	return nil
+}
+
 func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) {
 	s.mx.RLock()
 	defer s.mx.RUnlock()
@@ -204,16 +275,34 @@ func (s *BadgerMarkSet) tryDB(key []byte) (has bool, err error) {
 // writer holds the exclusive lock
 func (s *BadgerMarkSet) put(key string) (write bool, seqno int) {
 	s.pend[key] = struct{}{}
-	if len(s.pend) < badgerMarkSetBatchSize {
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
 		return false, 0
 	}

-	seqno = s.seqno
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) putMany(batch []cid.Cid) (write bool, seqno int) {
+	for _, c := range batch {
+		key := string(c.Hash())
+		s.pend[key] = struct{}{}
+	}
+
+	if !s.persist && len(s.pend) < badgerMarkSetBatchSize {
+		return false, 0
+	}
+
+	seqno = s.nextBatch()
+	return true, seqno
+}
+
+func (s *BadgerMarkSet) nextBatch() int {
+	seqno := s.seqno
 	s.seqno++
 	s.writing[seqno] = s.pend
 	s.pend = make(map[string]struct{})
-
-	return true, seqno
+	return seqno
 }

 func (s *BadgerMarkSet) write(seqno int) (err error) {
@@ -258,6 +347,14 @@ func (s *BadgerMarkSet) write(seqno int) (err error) {
 		return xerrors.Errorf("error flushing batch to badger markset: %w", err)
 	}

+	s.mx.RLock()
+	persist := s.persist
+	s.mx.RUnlock()
+
+	if persist {
+		return s.db.Sync()
+	}
+
 	return nil
 }

@@ -277,26 +374,29 @@ func (s *BadgerMarkSet) Close() error {
 	db := s.db
 	s.db = nil

-	return closeTransientBadgerDB(db, s.path)
+	return closeBadgerDB(db, s.path, s.persist)
 }

-func (s *BadgerMarkSet) SetConcurrent() {}
+func openBadgerDB(path string, recover bool) (*badger.DB, error) {
+	// if it is not a recovery, clean up first
+	if !recover {
+		err := os.RemoveAll(path)
+		if err != nil {
+			return nil, xerrors.Errorf("error clearing markset directory: %w", err)
+		}

-func openTransientBadgerDB(path string) (*badger.DB, error) {
-	// clean up first
-	err := os.RemoveAll(path)
-	if err != nil {
-		return nil, xerrors.Errorf("error clearing markset directory: %w", err)
-	}
-
-	err = os.MkdirAll(path, 0755) //nolint:gosec
-	if err != nil {
-		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		err = os.MkdirAll(path, 0755) //nolint:gosec
+		if err != nil {
+			return nil, xerrors.Errorf("error creating markset directory: %w", err)
+		}
 	}

 	opts := badger.DefaultOptions(path)
+	// we manually sync when we are in critical section
 	opts.SyncWrites = false
+	// no need to do that
 	opts.CompactL0OnClose = false
+	// we store hashes, not much to gain by compression
 	opts.Compression = options.None
 	// Note: We use FileIO for loading modes to avoid memory thrashing and interference
 	//       between the system blockstore and the markset.
@@ -305,6 +405,15 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	//       exceeded 1GB in size.
 	opts.TableLoadingMode = options.FileIO
 	opts.ValueLogLoadingMode = options.FileIO
+	// We increase the number of L0 tables before compaction to make it unlikely to
+	// be necessary.
+	opts.NumLevelZeroTables = 20      // default is 5
+	opts.NumLevelZeroTablesStall = 30 // default is 10
+	// increase the number of compactors from default 2 so that if we ever have to
+	// compact, it is fast
+	if runtime.NumCPU()/2 > opts.NumCompactors {
+		opts.NumCompactors = runtime.NumCPU() / 2
+	}
 	opts.Logger = &badgerLogger{
 		SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(),
 		skip2:         log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(),
@@ -313,12 +422,16 @@ func openTransientBadgerDB(path string) (*badger.DB, error) {
 	return badger.Open(opts)
 }

-func closeTransientBadgerDB(db *badger.DB, path string) error {
+func closeBadgerDB(db *badger.DB, path string, persist bool) error {
 	err := db.Close()
 	if err != nil {
 		return xerrors.Errorf("error closing badger markset: %w", err)
 	}

+	if persist {
+		return nil
+	}
+
 	err = os.RemoveAll(path)
 	if err != nil {
 		return xerrors.Errorf("error deleting badger markset: %w", err)
@@ -1,12 +1,20 @@
 package splitstore

 import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
 	"sync"

+	"golang.org/x/xerrors"
+
 	cid "github.com/ipfs/go-cid"
 )

-type MapMarkSetEnv struct{}
+type MapMarkSetEnv struct {
+	path string
+}

 var _ MarkSetEnv = (*MapMarkSetEnv)(nil)

@@ -14,55 +22,194 @@ type MapMarkSet struct {
 	mx  sync.RWMutex
 	set map[string]struct{}

-	ts bool
+	persist bool
+	file    *os.File
+	buf     *bufio.Writer
+
+	path string
 }

 var _ MarkSet = (*MapMarkSet)(nil)
-var _ MarkSetVisitor = (*MapMarkSet)(nil)

-func NewMapMarkSetEnv() (*MapMarkSetEnv, error) {
-	return &MapMarkSetEnv{}, nil
+func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) {
+	msPath := filepath.Join(path, "markset.map")
+	err := os.MkdirAll(msPath, 0755) //nolint:gosec
+	if err != nil {
+		return nil, xerrors.Errorf("error creating markset directory: %w", err)
+	}
+
+	return &MapMarkSetEnv{path: msPath}, nil
 }

-func (e *MapMarkSetEnv) create(name string, sizeHint int64) (*MapMarkSet, error) {
+func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
 	return &MapMarkSet{
-		set: make(map[string]struct{}, sizeHint),
+		set:  make(map[string]struct{}, sizeHint),
+		path: path,
 	}, nil
 }

-func (e *MapMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) {
-	return e.create(name, sizeHint)
-}
+func (e *MapMarkSetEnv) Recover(name string) (MarkSet, error) {
+	path := filepath.Join(e.path, name)
+	s := &MapMarkSet{
+		set:  make(map[string]struct{}),
+		path: path,
+	}

-func (e *MapMarkSetEnv) CreateVisitor(name string, sizeHint int64) (MarkSetVisitor, error) {
-	return e.create(name, sizeHint)
-}
+	in, err := os.Open(path)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for read: %w", err)
+	}
+	defer in.Close() //nolint:errcheck

-func (e *MapMarkSetEnv) SupportsVisitor() bool { return true }
+	// wrap a buffered reader to make this faster
+	buf := bufio.NewReader(in)
+	for {
+		var sz byte
+		if sz, err = buf.ReadByte(); err != nil {
+			break
+		}
+
+		key := make([]byte, int(sz))
+		if _, err = io.ReadFull(buf, key); err != nil {
+			break
+		}
+
+		s.set[string(key)] = struct{}{}
+	}
+
+	if err != io.EOF {
+		return nil, xerrors.Errorf("error reading markset file: %w", err)
+	}
+
+	file, err := os.OpenFile(s.path, os.O_WRONLY|os.O_APPEND, 0)
+	if err != nil {
+		return nil, xerrors.Errorf("error opening markset file for write: %w", err)
+	}
+
+	s.persist = true
+	s.file = file
+	s.buf = bufio.NewWriter(file)
+
+	return s, nil
+}

 func (e *MapMarkSetEnv) Close() error {
 	return nil
 }

-func (s *MapMarkSet) Mark(cid cid.Cid) error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+func (s *MapMarkSet) BeginCriticalSection() error {
+	s.mx.Lock()
+	defer s.mx.Unlock()

 	if s.set == nil {
 		return errMarkSetClosed
 	}

-	s.set[string(cid.Hash())] = struct{}{}
+	if s.persist {
+		return nil
+	}
+
+	file, err := os.OpenFile(s.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return xerrors.Errorf("error opening markset file: %w", err)
+	}
+
+	// wrap a buffered writer to make this faster
+	s.buf = bufio.NewWriter(file)
+	for key := range s.set {
+		if err := s.writeKey([]byte(key), false); err != nil {
+			_ = file.Close()
+			s.buf = nil
+			return err
+		}
+	}
+	if err := s.buf.Flush(); err != nil {
+		_ = file.Close()
+		s.buf = nil
+		return xerrors.Errorf("error flushing markset file buffer: %w", err)
+	}
+
+	s.file = file
+	s.persist = true
+
+	return nil
+}
+
+func (s *MapMarkSet) EndCriticalSection() {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if !s.persist {
+		return
+	}
+
+	_ = s.file.Close()
+	_ = os.Remove(s.path)
+	s.file = nil
+	s.buf = nil
+	s.persist = false
+}
+
+func (s *MapMarkSet) Mark(c cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	hash := c.Hash()
+	s.set[string(hash)] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return err
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (s *MapMarkSet) MarkMany(batch []cid.Cid) error {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return errMarkSetClosed
+	}
+
+	for _, c := range batch {
+		hash := c.Hash()
+		s.set[string(hash)] = struct{}{}
+
+		if s.persist {
+			if err := s.writeKey(hash, false); err != nil {
+				return err
+			}
+		}
+	}
+
+	if s.persist {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+
+		if err := s.file.Sync(); err != nil {
+			return xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return nil
 }

 func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.RLock()
-		defer s.mx.RUnlock()
-	}
+	s.mx.RLock()
+	defer s.mx.RUnlock()

 	if s.set == nil {
 		return false, errMarkSetClosed
@@ -73,33 +220,70 @@ func (s *MapMarkSet) Has(cid cid.Cid) (bool, error) {
 }

 func (s *MapMarkSet) Visit(c cid.Cid) (bool, error) {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
-	}
+	s.mx.Lock()
+	defer s.mx.Unlock()

 	if s.set == nil {
 		return false, errMarkSetClosed
 	}

-	key := string(c.Hash())
+	hash := c.Hash()
+	key := string(hash)
 	if _, ok := s.set[key]; ok {
 		return false, nil
 	}

 	s.set[key] = struct{}{}
+
+	if s.persist {
+		if err := s.writeKey(hash, true); err != nil {
+			return false, err
+		}
+		if err := s.file.Sync(); err != nil {
+			return false, xerrors.Errorf("error syncing markset: %w", err)
+		}
+	}
+
 	return true, nil
 }

 func (s *MapMarkSet) Close() error {
-	if s.ts {
-		s.mx.Lock()
-		defer s.mx.Unlock()
+	s.mx.Lock()
+	defer s.mx.Unlock()
+
+	if s.set == nil {
+		return nil
 	}
+
 	s.set = nil
+
+	if s.file != nil {
+		if err := s.file.Close(); err != nil {
+			log.Warnf("error closing markset file: %s", err)
+		}
+
+		if !s.persist {
+			if err := os.Remove(s.path); err != nil {
+				log.Warnf("error removing markset file: %s", err)
+			}
+		}
+	}
+
 	return nil
 }

-func (s *MapMarkSet) SetConcurrent() {
-	s.ts = true
+func (s *MapMarkSet) writeKey(k []byte, flush bool) error {
+	if err := s.buf.WriteByte(byte(len(k))); err != nil {
+		return xerrors.Errorf("error writing markset key length to disk: %w", err)
+	}
+	if _, err := s.buf.Write(k); err != nil {
+		return xerrors.Errorf("error writing markset key to disk: %w", err)
+	}
+	if flush {
+		if err := s.buf.Flush(); err != nil {
+			return xerrors.Errorf("error flushing markset buffer to disk: %w", err)
+		}
+	}
+
+	return nil
 }
@@ -11,7 +11,10 @@ import (

 func TestMapMarkSet(t *testing.T) {
 	testMarkSet(t, "map")
+	testMarkSetRecovery(t, "map")
+	testMarkSetMarkMany(t, "map")
 	testMarkSetVisitor(t, "map")
+	testMarkSetVisitorRecovery(t, "map")
 }

 func TestBadgerMarkSet(t *testing.T) {
@@ -21,12 +24,13 @@ func TestBadgerMarkSet(t *testing.T) {
 		badgerMarkSetBatchSize = bs
 	})
 	testMarkSet(t, "badger")
+	testMarkSetRecovery(t, "badger")
+	testMarkSetMarkMany(t, "badger")
 	testMarkSetVisitor(t, "badger")
+	testMarkSetVisitorRecovery(t, "badger")
 }

 func testMarkSet(t *testing.T, lsType string) {
-	t.Helper()
-
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -42,12 +46,12 @@ func testMarkSet(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	hotSet, err := env.Create("hot", 0)
+	hotSet, err := env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err := env.Create("cold", 0)
+	coldSet, err := env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,6 +66,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}

 	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -73,6 +78,7 @@ func testMarkSet(t *testing.T, lsType string) {
 	}

 	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
 		has, err := s.Has(cid)
 		if err != nil {
 			t.Fatal(err)
@@ -114,12 +120,12 @@ func testMarkSet(t *testing.T, lsType string) {
 		t.Fatal(err)
 	}

-	hotSet, err = env.Create("hot", 0)
+	hotSet, err = env.New("hot", 0)
 	if err != nil {
 		t.Fatal(err)
 	}

-	coldSet, err = env.Create("cold", 0)
+	coldSet, err = env.New("cold", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -150,8 +156,6 @@ func testMarkSet(t *testing.T, lsType string) {
 }

 func testMarkSetVisitor(t *testing.T, lsType string) {
-	t.Helper()
-
 	path, err := ioutil.TempDir("", "markset.*")
 	if err != nil {
 		t.Fatal(err)
@@ -167,7 +171,7 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	}
 	defer env.Close() //nolint:errcheck

-	visitor, err := env.CreateVisitor("test", 0)
+	visitor, err := env.New("test", 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,3 +223,322 @@ func testMarkSetVisitor(t *testing.T, lsType string) {
 	mustNotVisit(visitor, k3)
 	mustNotVisit(visitor, k4)
 }
+
+func testMarkSetVisitorRecovery(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	visitor, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer visitor.Close() //nolint:errcheck
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !visit {
+			t.Fatal("object should be visited")
+		}
+	}
+
+	mustNotVisit := func(v ObjectVisitor, cid cid.Cid) {
+		visit, err := v.Visit(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if visit {
+			t.Fatal("unexpected visit")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	mustVisit(visitor, k1)
+	mustVisit(visitor, k2)
+
+	if err := visitor.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	mustVisit(visitor, k3)
+	mustVisit(visitor, k4)
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	visitor, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustNotVisit(visitor, k1)
+	mustNotVisit(visitor, k2)
+	mustNotVisit(visitor, k3)
+	mustNotVisit(visitor, k4)
+
+	visitor.EndCriticalSection()
+
+	if err := visitor.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetRecovery(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.Mark(k1); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k2); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.Mark(k3); err != nil {
+		t.Fatal(err)
+	}
+	if err := markSet.Mark(k4); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
+
+func testMarkSetMarkMany(t *testing.T, lsType string) {
+	path, err := ioutil.TempDir("", "markset.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	env, err := OpenMarkSetEnv(path, lsType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer env.Close() //nolint:errcheck
+
+	markSet, err := env.New("test", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	makeCid := func(key string) cid.Cid {
+		h, err := multihash.Sum([]byte(key), multihash.SHA2_256, -1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return cid.NewCidV1(cid.Raw, h)
+	}
+
+	mustHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatal("mark not found")
+		}
+	}
+
+	mustNotHave := func(s MarkSet, cid cid.Cid) {
+		t.Helper()
+		has, err := s.Has(cid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("unexpected mark")
+		}
+	}
+
+	k1 := makeCid("a")
+	k2 := makeCid("b")
+	k3 := makeCid("c")
+	k4 := makeCid("d")
+
+	if err := markSet.MarkMany([]cid.Cid{k1, k2}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustNotHave(markSet, k3)
+	mustNotHave(markSet, k4)
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := markSet.MarkMany([]cid.Cid{k3, k4}); err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	markSet, err = env.Recover("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mustHave(markSet, k1)
+	mustHave(markSet, k2)
+	mustHave(markSet, k3)
+	mustHave(markSet, k4)
+
+	markSet.EndCriticalSection()
+
+	if err := markSet.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = env.Recover("test")
+	if err == nil {
+		t.Fatal("expected recovery to fail")
+	}
+}
@@ -129,8 +129,6 @@ type SplitStore struct {

 	headChangeMx sync.Mutex

-	coldPurgeSize int
-
 	chain ChainAccessor
 	ds    dstore.Datastore
 	cold  bstore.Blockstore
@@ -158,6 +156,17 @@ type SplitStore struct {
 	txnRefsMx       sync.Mutex
 	txnRefs         map[cid.Cid]struct{}
 	txnMissing      map[cid.Cid]struct{}
+	txnMarkSet      MarkSet
+	txnSyncMx       sync.Mutex
+	txnSyncCond     sync.Cond
+	txnSync         bool
+
+	// background cold object reification
+	reifyWorkers    sync.WaitGroup
+	reifyMx         sync.Mutex
+	reifyCond       sync.Cond
+	reifyPend       map[cid.Cid]struct{}
+	reifyInProgress map[cid.Cid]struct{}

 	// registered protectors
 	protectors []func(func(cid.Cid) error) error
@@ -186,10 +195,6 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		return nil, err
 	}

-	if !markSetEnv.SupportsVisitor() {
-		return nil, xerrors.Errorf("markset type does not support atomic visitors")
-	}
-
 	// and now we can make a SplitStore
 	ss := &SplitStore{
 		cfg:        cfg,
@@ -198,13 +203,16 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		cold:       cold,
 		hot:        hots,
 		markSetEnv: markSetEnv,
-
-		coldPurgeSize: defaultColdPurgeSize,
 	}

 	ss.txnViewsCond.L = &ss.txnViewsMx
+	ss.txnSyncCond.L = &ss.txnSyncMx
 	ss.ctx, ss.cancel = context.WithCancel(context.Background())

+	ss.reifyCond.L = &ss.reifyMx
+	ss.reifyPend = make(map[cid.Cid]struct{})
+	ss.reifyInProgress = make(map[cid.Cid]struct{})
+
 	if enableDebugLog {
 		ss.debug, err = openDebugLog(path)
 		if err != nil {
@@ -212,6 +220,14 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co
 		}
 	}

+	if ss.checkpointExists() {
+		log.Info("found compaction checkpoint; resuming compaction")
+		if err := ss.completeCompaction(); err != nil {
+			markSetEnv.Close() //nolint:errcheck
+			return nil, xerrors.Errorf("error resuming compaction: %w", err)
+		}
+	}
+
 	return ss, nil
 }

@@ -234,6 +250,20 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return false, err
+		}
+
+		if has {
+			return s.has(cid)
+		}
+
+		return s.cold.Has(ctx, cid)
+	}
+
 	has, err := s.hot.Has(ctx, cid)

 	if err != nil {
@@ -245,7 +275,13 @@ func (s *SplitStore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
 		return true, nil
 	}

-	return s.cold.Has(ctx, cid)
+	has, err = s.cold.Has(ctx, cid)
+	if has && bstore.IsHotView(ctx) {
+		s.reifyColdObject(cid)
+	}
+
+	return has, err
+
 }

 func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
@@ -261,6 +297,20 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return nil, err
+		}
+
+		if has {
+			return s.get(cid)
+		}
+
+		return s.cold.Get(ctx, cid)
+	}
+
 	blk, err := s.hot.Get(ctx, cid)

 	switch err {
@@ -275,8 +325,11 @@ func (s *SplitStore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)

 		blk, err = s.cold.Get(ctx, cid)
 		if err == nil {
-			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}

+			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return blk, err

@@ -298,6 +351,20 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()

+	// critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		if err != nil {
+			return 0, err
+		}
+
+		if has {
+			return s.getSize(cid)
+		}
+
+		return s.cold.GetSize(ctx, cid)
+	}
+
 	size, err := s.hot.GetSize(ctx, cid)

 	switch err {
@@ -312,6 +379,10 @@ func (s *SplitStore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {

 		size, err = s.cold.GetSize(ctx, cid)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return size, err
@@ -336,6 +407,12 @@ func (s *SplitStore) Put(ctx context.Context, blk blocks.Block) error {

 	s.debug.LogWrite(blk)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs([]cid.Cid{blk.Cid()})
+		return nil
+	}
+
 	s.trackTxnRef(blk.Cid())
 	return nil
 }
@@ -381,6 +458,12 @@ func (s *SplitStore) PutMany(ctx context.Context, blks []blocks.Block) error {

 	s.debug.LogWriteMany(blks)

+	// critical section
+	if s.txnMarkSet != nil {
+		s.markLiveRefs(batch)
+		return nil
+	}
+
 	s.trackTxnRefMany(batch)
 	return nil
 }
@@ -440,6 +523,23 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro
 		return cb(data)
 	}

+	// critical section
+	s.txnLk.RLock() // the lock is released in protectView if we are not in critical section
+	if s.txnMarkSet != nil {
+		has, err := s.txnMarkSet.Has(cid)
+		s.txnLk.RUnlock()
+
+		if err != nil {
+			return err
+		}
+
+		if has {
+			return s.view(cid, cb)
+		}
+
+		return s.cold.View(ctx, cid, cb)
+	}
+
 	// views are (optimistically) protected two-fold:
 	// - if there is an active transaction, then the reference is protected.
 	// - if there is no active transaction, active views are tracked in a
@@ -460,6 +560,10 @@ func (s *SplitStore) View(ctx context.Context, cid cid.Cid, cb func([]byte) erro

 		err = s.cold.View(ctx, cid, cb)
 		if err == nil {
+			if bstore.IsHotView(ctx) {
+				s.reifyColdObject(cid)
+			}
+
 			stats.Record(s.ctx, metrics.SplitstoreMiss.M(1))
 		}
 		return err
@@ -569,6 +673,9 @@ func (s *SplitStore) Start(chain ChainAccessor, us stmgr.UpgradeSchedule) error
 		}
 	}

+	// spawn the reifier
+	go s.reifyOrchestrator()
+
 	// watch the chain
 	chain.SubscribeHeadChanges(s.HeadChange)

@@ -589,12 +696,19 @@ func (s *SplitStore) Close() error {
 	}

 	if atomic.LoadInt32(&s.compacting) == 1 {
+		s.txnSyncMx.Lock()
+		s.txnSync = true
+		s.txnSyncCond.Broadcast()
+		s.txnSyncMx.Unlock()
+
 		log.Warn("close with ongoing compaction in progress; waiting for it to finish...")
 		for atomic.LoadInt32(&s.compacting) == 1 {
 			time.Sleep(time.Second)
 		}
 	}

+	s.reifyCond.Broadcast()
+	s.reifyWorkers.Wait()
 	s.cancel()
 	return multierr.Combine(s.markSetEnv.Close(), s.debug.Close())
 }
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"sync/atomic"
 	"time"

@@ -67,7 +68,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	}
 	defer output.Close() //nolint:errcheck

+	var mx sync.Mutex
 	write := func(format string, args ...interface{}) {
+		mx.Lock()
+		defer mx.Unlock()
 		_, err := fmt.Fprintf(output, format+"\n", args...)
 		if err != nil {
 			log.Warnf("error writing check output: %s", err)
@@ -82,9 +86,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 	write("compaction index: %d", s.compactionIndex)
 	write("--")

-	var coldCnt, missingCnt int64
+	coldCnt := new(int64)
+	missingCnt := new(int64)

-	visitor, err := s.markSetEnv.CreateVisitor("check", 0)
+	visitor, err := s.markSetEnv.New("check", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -111,10 +116,10 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 			}

 			if has {
-				coldCnt++
+				atomic.AddInt64(coldCnt, 1)
 				write("cold object reference: %s", c)
 			} else {
-				missingCnt++
+				atomic.AddInt64(missingCnt, 1)
 				write("missing object reference: %s", c)
 				return errStopWalk
 			}
@@ -128,9 +133,9 @@ func (s *SplitStore) doCheck(curTs *types.TipSet) error {
 		return err
 	}

-	log.Infow("check done", "cold", coldCnt, "missing", missingCnt)
+	log.Infow("check done", "cold", *coldCnt, "missing", *missingCnt)
 	write("--")
-	write("cold: %d missing: %d", coldCnt, missingCnt)
+	write("cold: %d missing: %d", *coldCnt, *missingCnt)
 	write("DONE")

 	return nil
@@ -3,8 +3,10 @@ package splitstore
 import (
 	"bytes"
 	"errors"
+	"os"
+	"path/filepath"
 	"runtime"
-	"sort"
+	"sync"
 	"sync/atomic"
 	"time"

@@ -47,6 +49,10 @@ var (
 	// SyncGapTime is the time delay from a tipset's min timestamp before we decide
 	// there is a sync gap
 	SyncGapTime = time.Minute
+
+	// SyncWaitTime is the time delay from a tipset's min timestamp before we decide
+	// we have synced.
+	SyncWaitTime = 30 * time.Second
 )

 var (
@@ -56,8 +62,6 @@ var (

 const (
 	batchSize = 16384
-
-	defaultColdPurgeSize = 7_000_000
 )

 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -140,9 +144,9 @@ func (s *SplitStore) isNearUpgrade(epoch abi.ChainEpoch) bool {
 // transactionally protect incoming tipsets
 func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 	s.txnLk.RLock()
-	defer s.txnLk.RUnlock()

 	if !s.txnActive {
+		s.txnLk.RUnlock()
 		return
 	}

@@ -151,12 +155,115 @@ func (s *SplitStore) protectTipSets(apply []*types.TipSet) {
 		cids = append(cids, ts.Cids()...)
 	}

+	if len(cids) == 0 {
+		s.txnLk.RUnlock()
+		return
+	}
+
+	// critical section
+	if s.txnMarkSet != nil {
+		curTs := apply[len(apply)-1]
+		timestamp := time.Unix(int64(curTs.MinTimestamp()), 0)
+		doSync := time.Since(timestamp) < SyncWaitTime
+		go func() {
+			if doSync {
+				defer func() {
+					s.txnSyncMx.Lock()
+					defer s.txnSyncMx.Unlock()
+					s.txnSync = true
+					s.txnSyncCond.Broadcast()
+				}()
+			}
+			defer s.txnLk.RUnlock()
+			s.markLiveRefs(cids)
+
+		}()
+		return
+	}
+
 	s.trackTxnRefMany(cids)
+	s.txnLk.RUnlock()
+}
+
+func (s *SplitStore) markLiveRefs(cids []cid.Cid) {
+	log.Debugf("marking %d live refs", len(cids))
+	startMark := time.Now()
+
+	count := new(int32)
+	visitor := newConcurrentVisitor()
+	walkObject := func(c cid.Cid) error {
+		return s.walkObjectIncomplete(c, visitor,
+			func(c cid.Cid) error {
+				if isUnitaryObject(c) {
+					return errStopWalk
+				}
+
+				visit, err := s.txnMarkSet.Visit(c)
+				if err != nil {
+					return xerrors.Errorf("error visiting object: %w", err)
+				}
+
+				if !visit {
+					return errStopWalk
+				}
+
+				atomic.AddInt32(count, 1)
+				return nil
+			},
+			func(missing cid.Cid) error {
+				log.Warnf("missing object reference %s in %s", missing, c)
+				return errStopWalk
+			})
+	}
+
+	// optimize the common case of single put
+	if len(cids) == 1 {
+		if err := walkObject(cids[0]); err != nil {
+			log.Errorf("error marking tipset refs: %s", err)
+		}
+		log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
+		return
+	}
+
+	workch := make(chan cid.Cid, len(cids))
+	for _, c := range cids {
+		workch <- c
+	}
+	close(workch)
+
+	worker := func() error {
+		for c := range workch {
+			if err := walkObject(c); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	workers := runtime.NumCPU() / 2
+	if workers < 2 {
+		workers = 2
+	}
+	if workers > len(cids) {
+		workers = len(cids)
+	}
+
+	g := new(errgroup.Group)
+	for i := 0; i < workers; i++ {
+		g.Go(worker)
+	}
+
+	if err := g.Wait(); err != nil {
+		log.Errorf("error marking tipset refs: %s", err)
+	}
+
+	log.Debugw("marking live refs done", "took", time.Since(startMark), "marked", *count)
 }

 // transactionally protect a view
 func (s *SplitStore) protectView(c cid.Cid) {
-	s.txnLk.RLock()
+	//  the txnLk is held for read
 	defer s.txnLk.RUnlock()

 	if s.txnActive {
@@ -227,7 +334,7 @@ func (s *SplitStore) trackTxnRefMany(cids []cid.Cid) {
 }

 // protect all pending transactional references
-func (s *SplitStore) protectTxnRefs(markSet MarkSetVisitor) error {
+func (s *SplitStore) protectTxnRefs(markSet MarkSet) error {
 	for {
 		var txnRefs map[cid.Cid]struct{}

@@ -299,14 +406,14 @@ func (s *SplitStore) protectTxnRefs(markSet MarkSetVisitor) error {

 // transactionally protect a reference by walking the object and marking.
 // concurrent markings are short circuited by checking the markset.
-func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSetVisitor) error {
+func (s *SplitStore) doTxnProtect(root cid.Cid, markSet MarkSet) error {
 	if err := s.checkClosing(); err != nil {
 		return err
 	}

 	// Note: cold objects are deleted heaviest first, so the consituents of an object
 	// cannot be deleted before the object itself.
-	return s.walkObjectIncomplete(root, tmpVisitor(),
+	return s.walkObjectIncomplete(root, newTmpVisitor(),
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
 				return errStopWalk
@@ -386,6 +493,12 @@ func (s *SplitStore) compact(curTs *types.TipSet) {
 }

 func (s *SplitStore) doCompact(curTs *types.TipSet) error {
+	if s.checkpointExists() {
+		// this really shouldn't happen, but if it somehow does, it means that the hotstore
+		// might be potentially inconsistent; abort compaction and notify the user to intervene.
+		return xerrors.Errorf("checkpoint exists; aborting compaction")
+	}
+
 	currentEpoch := curTs.Height()
 	boundaryEpoch := currentEpoch - CompactionBoundary

@@ -397,7 +510,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {

 	log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "inclMsgsEpoch", inclMsgsEpoch, "compactionIndex", s.compactionIndex)

-	markSet, err := s.markSetEnv.CreateVisitor("live", s.markSetSize)
+	markSet, err := s.markSetEnv.New("live", s.markSetSize)
 	if err != nil {
 		return xerrors.Errorf("error creating mark set: %w", err)
 	}
@@ -408,9 +521,6 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}

-	// we are ready for concurrent marking
-	s.beginTxnMarking(markSet)
-
 	// 0. track all protected references at beginning of compaction; anything added later should
 	//    be transactionally protected by the write
 	log.Info("protecting references with registered protectors")
@@ -424,7 +534,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("marking reachable objects")
 	startMark := time.Now()

-	var count int64
+	count := new(int64)
 	err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, &noopVisitor{},
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
@@ -440,7 +550,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 				return errStopWalk
 			}

-			count++
+			atomic.AddInt64(count, 1)
 			return nil
 		})

@@ -448,9 +558,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error marking: %w", err)
 	}

-	s.markSetSize = count + count>>2 // overestimate a bit
+	s.markSetSize = *count + *count>>2 // overestimate a bit

-	log.Infow("marking done", "took", time.Since(startMark), "marked", count)
+	log.Infow("marking done", "took", time.Since(startMark), "marked", *count)

 	if err := s.checkClosing(); err != nil {
 		return err
@@ -470,10 +580,15 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	log.Info("collecting cold objects")
 	startCollect := time.Now()

+	coldw, err := NewColdSetWriter(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error creating coldset: %w", err)
+	}
+	defer coldw.Close() //nolint:errcheck
+
 	// some stats for logging
 	var hotCnt, coldCnt int

-	cold := make([]cid.Cid, 0, s.coldPurgeSize)
 	err = s.hot.ForEachKey(func(c cid.Cid) error {
 		// was it marked?
 		mark, err := markSet.Has(c)
@@ -487,7 +602,9 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		}

 		// it's cold, mark it as candidate for move
-		cold = append(cold, c)
+		if err := coldw.Write(c); err != nil {
+			return xerrors.Errorf("error writing cid to coldstore: %w", err)
+		}
 		coldCnt++

 		return nil
@@ -497,12 +614,12 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error collecting cold objects: %w", err)
 	}

-	log.Infow("cold collection done", "took", time.Since(startCollect))
-
-	if coldCnt > 0 {
-		s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
+	if err := coldw.Close(); err != nil {
+		return xerrors.Errorf("error closing coldset: %w", err)
 	}

+	log.Infow("cold collection done", "took", time.Since(startCollect))
+
 	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
 	stats.Record(s.ctx, metrics.SplitstoreCompactionHot.M(int64(hotCnt)))
 	stats.Record(s.ctx, metrics.SplitstoreCompactionCold.M(int64(coldCnt)))
@@ -520,11 +637,17 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return err
 	}

+	coldr, err := NewColdSetReader(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening coldset: %w", err)
+	}
+	defer coldr.Close() //nolint:errcheck
+
 	// 3. copy the cold objects to the coldstore -- if we have one
 	if !s.cfg.DiscardColdBlocks {
 		log.Info("moving cold objects to the coldstore")
 		startMove := time.Now()
-		err = s.moveColdBlocks(cold)
+		err = s.moveColdBlocks(coldr)
 		if err != nil {
 			return xerrors.Errorf("error moving cold objects: %w", err)
 		}
@@ -533,41 +656,64 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
+
+		if err := coldr.Reset(); err != nil {
+			return xerrors.Errorf("error resetting coldset: %w", err)
+		}
 	}

-	// 4. sort cold objects so that the dags with most references are deleted first
-	//    this ensures that we can't refer to a dag with its consituents already deleted, ie
-	//    we lave no dangling references.
-	log.Info("sorting cold objects")
-	startSort := time.Now()
-	err = s.sortObjects(cold)
-	if err != nil {
-		return xerrors.Errorf("error sorting objects: %w", err)
-	}
-	log.Infow("sorting done", "took", time.Since(startSort))
-
-	// 4.1 protect transactional refs once more
-	//     strictly speaking, this is not necessary as purge will do it before deleting each
-	//     batch.  however, there is likely a largish number of references accumulated during
-	//     ths sort and this protects before entering pruge context.
-	err = s.protectTxnRefs(markSet)
-	if err != nil {
-		return xerrors.Errorf("error protecting transactional refs: %w", err)
+	// 4. Purge cold objects with checkpointing for recovery.
+	// This is the critical section of compaction, whereby any cold object not in the markSet is
+	// considered already deleted.
+	// We delete cold objects in batches, holding the transaction lock, where we check the markSet
+	// again for new references created by the VM.
+	// After each batch, we write a checkpoint to disk; if the process is interrupted before completion,
+	// the process will continue from the checkpoint in the next recovery.
+	if err := s.beginCriticalSection(markSet); err != nil {
+		return xerrors.Errorf("error beginning critical section: %w", err)
 	}

 	if err := s.checkClosing(); err != nil {
 		return err
 	}

+	// wait for the head to catch up so that the current tipset is marked
+	s.waitForSync()
+
+	if err := s.checkClosing(); err != nil {
+		return err
+	}
+
+	checkpoint, err := NewCheckpoint(s.checkpointPath())
+	if err != nil {
+		return xerrors.Errorf("error creating checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
 	// 5. purge cold objects from the hotstore, taking protected references into account
 	log.Info("purging cold objects from the hotstore")
 	startPurge := time.Now()
-	err = s.purge(cold, markSet)
+	err = s.purge(coldr, checkpoint, markSet)
 	if err != nil {
-		return xerrors.Errorf("error purging cold blocks: %w", err)
+		return xerrors.Errorf("error purging cold objects: %w", err)
 	}
 	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))

+	s.endCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.checkpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := coldr.Close(); err != nil {
+		log.Warnf("error closing coldset: %s", err)
+	}
+	if err := os.Remove(s.coldSetPath()); err != nil {
+		log.Warnf("error removing coldset: %s", err)
+	}
+
 	// we are done; do some housekeeping
 	s.endTxnProtect()
 	s.gcHotstore()
@@ -598,12 +744,51 @@ func (s *SplitStore) beginTxnProtect() {
 	defer s.txnLk.Unlock()

 	s.txnActive = true
+	s.txnSync = false
 	s.txnRefs = make(map[cid.Cid]struct{})
 	s.txnMissing = make(map[cid.Cid]struct{})
 }

-func (s *SplitStore) beginTxnMarking(markSet MarkSetVisitor) {
-	markSet.SetConcurrent()
+func (s *SplitStore) beginCriticalSection(markSet MarkSet) error {
+	log.Info("beginning critical section")
+
+	// do that once first to get the bulk before the markset is in critical section
+	if err := s.protectTxnRefs(markSet); err != nil {
+		return xerrors.Errorf("error protecting transactional references: %w", err)
+	}
+
+	if err := markSet.BeginCriticalSection(); err != nil {
+		return xerrors.Errorf("error beginning critical section for markset: %w", err)
+	}
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	s.txnMarkSet = markSet
+
+	// and do it again while holding the lock to mark references that might have been created
+	// in the meantime and avoid races of the type Has->txnRef->enterCS->Get fails because
+	// it's not in the markset
+	if err := s.protectTxnRefs(markSet); err != nil {
+		return xerrors.Errorf("error protecting transactional references: %w", err)
+	}
+
+	return nil
+}
+
+func (s *SplitStore) waitForSync() {
+	log.Info("waiting for sync")
+	startWait := time.Now()
+	defer func() {
+		log.Infow("waiting for sync done", "took", time.Since(startWait))
+	}()
+
+	s.txnSyncMx.Lock()
+	defer s.txnSyncMx.Unlock()
+
+	for !s.txnSync {
+		s.txnSyncCond.Wait()
+	}
 }

 func (s *SplitStore) endTxnProtect() {
@@ -615,32 +800,51 @@ func (s *SplitStore) endTxnProtect() {
 	}

 	s.txnActive = false
+	s.txnSync = false
 	s.txnRefs = nil
 	s.txnMissing = nil
+	s.txnMarkSet = nil
+}
+
+func (s *SplitStore) endCriticalSection() {
+	log.Info("ending critical section")
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	s.txnMarkSet.EndCriticalSection()
+	s.txnMarkSet = nil
 }

 func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEpoch,
 	visitor ObjectVisitor, f func(cid.Cid) error) error {
-	var walked *cid.Set
-	toWalk := ts.Cids()
-	walkCnt := 0
-	scanCnt := 0
+	var walked ObjectVisitor
+	var mx sync.Mutex
+	// we copy the tipset first into a new slice, which allows us to reuse it in every epoch.
+	toWalk := make([]cid.Cid, len(ts.Cids()))
+	copy(toWalk, ts.Cids())
+	walkCnt := new(int64)
+	scanCnt := new(int64)

 	stopWalk := func(_ cid.Cid) error { return errStopWalk }

 	walkBlock := func(c cid.Cid) error {
-		if !walked.Visit(c) {
+		visit, err := walked.Visit(c)
+		if err != nil {
+			return err
+		}
+		if !visit {
 			return nil
 		}

-		walkCnt++
+		atomic.AddInt64(walkCnt, 1)

 		if err := f(c); err != nil {
 			return err
 		}

 		var hdr types.BlockHeader
-		err := s.view(c, func(data []byte) error {
+		err = s.view(c, func(data []byte) error {
 			return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
 		})

@@ -676,11 +880,13 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 			if err := s.walkObject(hdr.ParentStateRoot, visitor, f); err != nil {
 				return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
 			}
-			scanCnt++
+			atomic.AddInt64(scanCnt, 1)
 		}

 		if hdr.Height > 0 {
+			mx.Lock()
 			toWalk = append(toWalk, hdr.Parents...)
+			mx.Unlock()
 		}

 		return nil
@@ -692,20 +898,43 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 			return err
 		}

+		workers := len(toWalk)
+		if workers > runtime.NumCPU()/2 {
+			workers = runtime.NumCPU() / 2
+		}
+		if workers < 2 {
+			workers = 2
+		}
+
 		// the walk is BFS, so we can reset the walked set in every iteration and avoid building up
 		// a set that contains all blocks (1M epochs -> 5M blocks -> 200MB worth of memory and growing
 		// over time)
-		walked = cid.NewSet()
-		walking := toWalk
-		toWalk = nil
-		for _, c := range walking {
-			if err := walkBlock(c); err != nil {
-				return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
-			}
+		walked = newConcurrentVisitor()
+		workch := make(chan cid.Cid, len(toWalk))
+		for _, c := range toWalk {
+			workch <- c
+		}
+		close(workch)
+		toWalk = toWalk[:0]
+
+		g := new(errgroup.Group)
+		for i := 0; i < workers; i++ {
+			g.Go(func() error {
+				for c := range workch {
+					if err := walkBlock(c); err != nil {
+						return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
+					}
+				}
+				return nil
+			})
+		}
+
+		if err := g.Wait(); err != nil {
+			return err
 		}
 	}

-	log.Infow("chain walk done", "walked", walkCnt, "scanned", scanCnt)
+	log.Infow("chain walk done", "walked", *walkCnt, "scanned", *scanCnt)

 	return nil
 }
@@ -824,7 +1053,7 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
 	return nil
 }

-// internal version used by walk
+// internal version used during compaction and related operations
 func (s *SplitStore) view(c cid.Cid, cb func([]byte) error) error {
 	if isIdentiyCid(c) {
 		data, err := decodeIdentityCid(c)
@@ -859,10 +1088,34 @@ func (s *SplitStore) has(c cid.Cid) (bool, error) {
 	return s.cold.Has(s.ctx, c)
 }

-func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
+func (s *SplitStore) get(c cid.Cid) (blocks.Block, error) {
+	blk, err := s.hot.Get(s.ctx, c)
+	switch err {
+	case nil:
+		return blk, nil
+	case bstore.ErrNotFound:
+		return s.cold.Get(s.ctx, c)
+	default:
+		return nil, err
+	}
+}
+
+func (s *SplitStore) getSize(c cid.Cid) (int, error) {
+	sz, err := s.hot.GetSize(s.ctx, c)
+	switch err {
+	case nil:
+		return sz, nil
+	case bstore.ErrNotFound:
+		return s.cold.GetSize(s.ctx, c)
+	default:
+		return 0, err
+	}
+}
+
+func (s *SplitStore) moveColdBlocks(coldr *ColdSetReader) error {
 	batch := make([]blocks.Block, 0, batchSize)

-	for _, c := range cold {
+	err := coldr.ForEach(func(c cid.Cid) error {
 		if err := s.checkClosing(); err != nil {
 			return err
 		}
@@ -871,7 +1124,7 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 		if err != nil {
 			if err == bstore.ErrNotFound {
 				log.Warnf("hotstore missing block %s", c)
-				continue
+				return nil
 			}

 			return xerrors.Errorf("error retrieving block %s from hotstore: %w", c, err)
@@ -885,6 +1138,12 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 			}
 			batch = batch[:0]
 		}
+
+		return nil
+	})
+
+	if err != nil {
+		return xerrors.Errorf("error iterating coldset: %w", err)
 	}

 	if len(batch) > 0 {
@@ -897,177 +1156,202 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 	return nil
 }

-// sorts a slice of objects heaviest first -- it's a little expensive but worth the
-// guarantee that we don't leave dangling references behind, e.g. if we die in the middle
-// of a purge.
-func (s *SplitStore) sortObjects(cids []cid.Cid) error {
-	// we cache the keys to avoid making a gazillion of strings
-	keys := make(map[cid.Cid]string)
-	key := func(c cid.Cid) string {
-		s, ok := keys[c]
-		if !ok {
-			s = string(c.Hash())
-			keys[c] = s
-		}
-		return s
-	}
-
-	// compute sorting weights as the cumulative number of DAG links
-	weights := make(map[string]int)
-	for _, c := range cids {
-		// this can take quite a while, so check for shutdown with every opportunity
-		if err := s.checkClosing(); err != nil {
-			return err
-		}
-
-		w := s.getObjectWeight(c, weights, key)
-		weights[key(c)] = w
-	}
-
-	// sort!
-	sort.Slice(cids, func(i, j int) bool {
-		wi := weights[key(cids[i])]
-		wj := weights[key(cids[j])]
-		if wi == wj {
-			return bytes.Compare(cids[i].Hash(), cids[j].Hash()) > 0
-		}
-
-		return wi > wj
-	})
-
-	return nil
-}
-
-func (s *SplitStore) getObjectWeight(c cid.Cid, weights map[string]int, key func(cid.Cid) string) int {
-	w, ok := weights[key(c)]
-	if ok {
-		return w
-	}
-
-	// we treat block headers specially to avoid walking the entire chain
-	var hdr types.BlockHeader
-	err := s.view(c, func(data []byte) error {
-		return hdr.UnmarshalCBOR(bytes.NewBuffer(data))
-	})
-	if err == nil {
-		w1 := s.getObjectWeight(hdr.ParentStateRoot, weights, key)
-		weights[key(hdr.ParentStateRoot)] = w1
-
-		w2 := s.getObjectWeight(hdr.Messages, weights, key)
-		weights[key(hdr.Messages)] = w2
-
-		return 1 + w1 + w2
-	}
-
-	var links []cid.Cid
-	err = s.view(c, func(data []byte) error {
-		return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
-			links = append(links, c)
-		})
-	})
-	if err != nil {
-		return 1
-	}
-
-	w = 1
-	for _, c := range links {
-		// these are internal refs, so dags will be dags
-		if c.Prefix().Codec != cid.DagCBOR {
-			w++
-			continue
-		}
-
-		wc := s.getObjectWeight(c, weights, key)
-		weights[key(c)] = wc
-
-		w += wc
-	}
-
-	return w
-}
-
-func (s *SplitStore) purgeBatch(cids []cid.Cid, deleteBatch func([]cid.Cid) error) error {
-	if len(cids) == 0 {
-		return nil
-	}
-
-	// we don't delete one giant batch of millions of objects, but rather do smaller batches
-	// so that we don't stop the world for an extended period of time
-	done := false
-	for i := 0; !done; i++ {
-		start := i * batchSize
-		end := start + batchSize
-		if end >= len(cids) {
-			end = len(cids)
-			done = true
-		}
-
-		err := deleteBatch(cids[start:end])
-		if err != nil {
-			return xerrors.Errorf("error deleting batch: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSetVisitor) error {
+func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet MarkSet) error {
+	batch := make([]cid.Cid, 0, batchSize)
 	deadCids := make([]cid.Cid, 0, batchSize)
+
 	var purgeCnt, liveCnt int
 	defer func() {
 		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
 	}()

-	return s.purgeBatch(cids,
-		func(cids []cid.Cid) error {
-			deadCids := deadCids[:0]
+	deleteBatch := func() error {
+		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)

-			for {
-				if err := s.checkClosing(); err != nil {
-					return err
-				}
+		purgeCnt += pc
+		liveCnt += lc
+		batch = batch[:0]

-				s.txnLk.Lock()
-				if len(s.txnRefs) == 0 {
-					// keep the lock!
-					break
-				}
+		return err
+	}

-				// unlock and protect
-				s.txnLk.Unlock()
+	err := coldr.ForEach(func(c cid.Cid) error {
+		batch = append(batch, c)
+		if len(batch) == batchSize {
+			return deleteBatch()
+		}

-				err := s.protectTxnRefs(markSet)
-				if err != nil {
-					return xerrors.Errorf("error protecting transactional refs: %w", err)
-				}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if len(batch) > 0 {
+		return deleteBatch()
+	}
+
+	return nil
+}
+
+func (s *SplitStore) purgeBatch(batch, deadCids []cid.Cid, checkpoint *Checkpoint, markSet MarkSet) (purgeCnt int, liveCnt int, err error) {
+	if err := s.checkClosing(); err != nil {
+		return 0, 0, err
+	}
+
+	s.txnLk.Lock()
+	defer s.txnLk.Unlock()
+
+	for _, c := range batch {
+		has, err := markSet.Has(c)
+		if err != nil {
+			return 0, 0, xerrors.Errorf("error checking markset for liveness: %w", err)
+		}
+
+		if has {
+			liveCnt++
+			continue
+		}
+
+		deadCids = append(deadCids, c)
+	}
+
+	if len(deadCids) == 0 {
+		if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
+			return 0, 0, xerrors.Errorf("error setting checkpoint: %w", err)
+		}
+
+		return 0, liveCnt, nil
+	}
+
+	if err := s.hot.DeleteMany(s.ctx, deadCids); err != nil {
+		return 0, liveCnt, xerrors.Errorf("error purging cold objects: %w", err)
+	}
+
+	s.debug.LogDelete(deadCids)
+	purgeCnt = len(deadCids)
+
+	if err := checkpoint.Set(batch[len(batch)-1]); err != nil {
+		return purgeCnt, liveCnt, xerrors.Errorf("error setting checkpoint: %w", err)
+	}
+
+	return purgeCnt, liveCnt, nil
+}
+
+func (s *SplitStore) coldSetPath() string {
+	return filepath.Join(s.path, "coldset")
+}
+
+func (s *SplitStore) checkpointPath() string {
+	return filepath.Join(s.path, "checkpoint")
+}
+
+func (s *SplitStore) checkpointExists() bool {
+	_, err := os.Stat(s.checkpointPath())
+	return err == nil
+}
+
+func (s *SplitStore) completeCompaction() error {
+	checkpoint, last, err := OpenCheckpoint(s.checkpointPath())
+	if err != nil {
+		return xerrors.Errorf("error opening checkpoint: %w", err)
+	}
+	defer checkpoint.Close() //nolint:errcheck
+
+	coldr, err := NewColdSetReader(s.coldSetPath())
+	if err != nil {
+		return xerrors.Errorf("error opening coldset: %w", err)
+	}
+	defer coldr.Close() //nolint:errcheck
+
+	markSet, err := s.markSetEnv.Recover("live")
+	if err != nil {
+		return xerrors.Errorf("error recovering markset: %w", err)
+	}
+	defer markSet.Close() //nolint:errcheck
+
+	// PURGE
+	log.Info("purging cold objects from the hotstore")
+	startPurge := time.Now()
+	err = s.completePurge(coldr, checkpoint, last, markSet)
+	if err != nil {
+		return xerrors.Errorf("error purging cold objects: %w", err)
+	}
+	log.Infow("purging cold objects from hotstore done", "took", time.Since(startPurge))
+
+	markSet.EndCriticalSection()
+
+	if err := checkpoint.Close(); err != nil {
+		log.Warnf("error closing checkpoint: %s", err)
+	}
+	if err := os.Remove(s.checkpointPath()); err != nil {
+		log.Warnf("error removing checkpoint: %s", err)
+	}
+	if err := coldr.Close(); err != nil {
+		log.Warnf("error closing coldset: %s", err)
+	}
+	if err := os.Remove(s.coldSetPath()); err != nil {
+		log.Warnf("error removing coldset: %s", err)
+	}
+
+	// Note: at this point we can start the splitstore; a compaction should run on
+	//       the first head change, which will trigger gc on the hotstore.
+	//       We don't mind the second (back-to-back) compaction as the head will
+	//       have advanced during marking and coldset accumulation.
+	return nil
+}
+
+func (s *SplitStore) completePurge(coldr *ColdSetReader, checkpoint *Checkpoint, start cid.Cid, markSet MarkSet) error {
+	if !start.Defined() {
+		return s.purge(coldr, checkpoint, markSet)
+	}
+
+	seeking := true
+	batch := make([]cid.Cid, 0, batchSize)
+	deadCids := make([]cid.Cid, 0, batchSize)
+
+	var purgeCnt, liveCnt int
+	defer func() {
+		log.Infow("purged cold objects", "purged", purgeCnt, "live", liveCnt)
+	}()
+
+	deleteBatch := func() error {
+		pc, lc, err := s.purgeBatch(batch, deadCids, checkpoint, markSet)
+
+		purgeCnt += pc
+		liveCnt += lc
+		batch = batch[:0]
+
+		return err
+	}
+
+	err := coldr.ForEach(func(c cid.Cid) error {
+		if seeking {
+			if start.Equals(c) {
+				seeking = false
 			}

-			defer s.txnLk.Unlock()
-
-			for _, c := range cids {
-				live, err := markSet.Has(c)
-				if err != nil {
-					return xerrors.Errorf("error checking for liveness: %w", err)
-				}
-
-				if live {
-					liveCnt++
-					continue
-				}
-
-				deadCids = append(deadCids, c)
-			}
-
-			err := s.hot.DeleteMany(s.ctx, deadCids)
-			if err != nil {
-				return xerrors.Errorf("error purging cold objects: %w", err)
-			}
-
-			s.debug.LogDelete(deadCids)
-
-			purgeCnt += len(deadCids)
 			return nil
-		})
+		}
+
+		batch = append(batch, c)
+		if len(batch) == batchSize {
+			return deleteBatch()
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if len(batch) > 0 {
+		return deleteBatch()
+	}
+
+	return nil
 }

 // I really don't like having this code, but we seem to have some occasional DAG references with
@@ -1077,7 +1361,7 @@ func (s *SplitStore) purge(cids []cid.Cid, markSet MarkSetVisitor) error {
 // have this gem[TM].
 // My best guess is that they are parent message receipts or yet to be computed state roots; magik
 // thinks the cause may be block validation.
-func (s *SplitStore) waitForMissingRefs(markSet MarkSetVisitor) {
+func (s *SplitStore) waitForMissingRefs(markSet MarkSet) {
 	s.txnLk.Lock()
 	missing := s.txnMissing
 	s.txnMissing = nil
@@ -1106,7 +1390,7 @@ func (s *SplitStore) waitForMissingRefs(markSet MarkSetVisitor) {
 		}

 		towalk := missing
-		visitor := tmpVisitor()
+		visitor := newTmpVisitor()
 		missing = make(map[cid.Cid]struct{})

 		for c := range towalk {
@@ -0,0 +1,214 @@
+package splitstore
+
+import (
+	"errors"
+	"runtime"
+	"sync/atomic"
+
+	"golang.org/x/xerrors"
+
+	blocks "github.com/ipfs/go-block-format"
+	cid "github.com/ipfs/go-cid"
+)
+
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
+
+func (s *SplitStore) reifyColdObject(c cid.Cid) {
+	if !s.isWarm() {
+		return
+	}
+
+	if isUnitaryObject(c) {
+		return
+	}
+
+	s.reifyMx.Lock()
+	defer s.reifyMx.Unlock()
+
+	_, ok := s.reifyInProgress[c]
+	if ok {
+		return
+	}
+
+	s.reifyPend[c] = struct{}{}
+	s.reifyCond.Broadcast()
+}
+
+func (s *SplitStore) reifyOrchestrator() {
+	workers := runtime.NumCPU() / 4
+	if workers < 2 {
+		workers = 2
+	}
+
+	workch := make(chan cid.Cid, workers)
+	defer close(workch)
+
+	for i := 0; i < workers; i++ {
+		s.reifyWorkers.Add(1)
+		go s.reifyWorker(workch)
+	}
+
+	for {
+		s.reifyMx.Lock()
+		for len(s.reifyPend) == 0 && atomic.LoadInt32(&s.closing) == 0 {
+			s.reifyCond.Wait()
+		}
+
+		if atomic.LoadInt32(&s.closing) != 0 {
+			s.reifyMx.Unlock()
+			return
+		}
+
+		reifyPend := s.reifyPend
+		s.reifyPend = make(map[cid.Cid]struct{})
+		s.reifyMx.Unlock()
+
+		for c := range reifyPend {
+			select {
+			case workch <- c:
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (s *SplitStore) reifyWorker(workch chan cid.Cid) {
+	defer s.reifyWorkers.Done()
+	for c := range workch {
+		s.doReify(c)
+	}
+}
+
+func (s *SplitStore) doReify(c cid.Cid) {
+	var toreify, totrack, toforget []cid.Cid
+
+	defer func() {
+		s.reifyMx.Lock()
+		defer s.reifyMx.Unlock()
+
+		for _, c := range toreify {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range totrack {
+			delete(s.reifyInProgress, c)
+		}
+		for _, c := range toforget {
+			delete(s.reifyInProgress, c)
+		}
+	}()
+
+	s.txnLk.RLock()
+	defer s.txnLk.RUnlock()
+
+	count := 0
+	err := s.walkObjectIncomplete(c, newTmpVisitor(),
+		func(c cid.Cid) error {
+			if isUnitaryObject(c) {
+				return errStopWalk
+			}
+
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
+			s.reifyMx.Lock()
+			_, inProgress := s.reifyInProgress[c]
+			if !inProgress {
+				s.reifyInProgress[c] = struct{}{}
+			}
+			s.reifyMx.Unlock()
+
+			if inProgress {
+				return errStopWalk
+			}
+
+			has, err := s.hot.Has(s.ctx, c)
+			if err != nil {
+				return xerrors.Errorf("error checking hotstore: %w", err)
+			}
+
+			if has {
+				if s.txnMarkSet != nil {
+					hasMark, err := s.txnMarkSet.Has(c)
+					if err != nil {
+						log.Warnf("error checking markset: %s", err)
+					} else if hasMark {
+						toforget = append(toforget, c)
+						return errStopWalk
+					}
+				} else {
+					totrack = append(totrack, c)
+					return errStopWalk
+				}
+			}
+
+			toreify = append(toreify, c)
+			return nil
+		},
+		func(missing cid.Cid) error {
+			log.Warnf("missing reference while reifying %s: %s", c, missing)
+			return errStopWalk
+		})
+
+	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
+		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
+		return
+	}
+
+	log.Debugf("reifying %d objects rooted at %s", len(toreify), c)
+
+	// this should not get too big, maybe some 100s of objects.
+	batch := make([]blocks.Block, 0, len(toreify))
+	for _, c := range toreify {
+		blk, err := s.cold.Get(s.ctx, c)
+		if err != nil {
+			log.Warnf("error retrieving cold object for reification (cid: %s): %s", c, err)
+			continue
+		}
+
+		if err := s.checkClosing(); err != nil {
+			return
+		}
+
+		batch = append(batch, blk)
+	}
+
+	if len(batch) > 0 {
+		err = s.hot.PutMany(s.ctx, batch)
+		if err != nil {
+			log.Warnf("error reifying cold object (cid: %s): %s", c, err)
+			return
+		}
+	}
+
+	if s.txnMarkSet != nil {
+		if len(toreify) > 0 {
+			if err := s.txnMarkSet.MarkMany(toreify); err != nil {
+				log.Warnf("error marking reified objects: %s", err)
+			}
+		}
+		if len(totrack) > 0 {
+			if err := s.txnMarkSet.MarkMany(totrack); err != nil {
+				log.Warnf("error marking tracked objects: %s", err)
+			}
+		}
+	} else {
+		// if txnActive is false these are noops
+		if len(toreify) > 0 {
+			s.trackTxnRefMany(toreify)
+		}
+		if len(totrack) > 0 {
+			s.trackTxnRefMany(totrack)
+		}
+	}
+}
@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -20,12 +23,14 @@ import (
 	datastore "github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	logging "github.com/ipfs/go-log/v2"
+	mh "github.com/multiformats/go-multihash"
 )

 func init() {
 	CompactionThreshold = 5
 	CompactionBoundary = 2
 	WarmupBoundary = 0
+	SyncWaitTime = time.Millisecond
 	logging.SetLogLevel("splitstore", "DEBUG")
 }

@@ -80,8 +85,17 @@ func testSplitStore(t *testing.T, cfg *Config) {
 		t.Fatal(err)
 	}

+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, cfg)
+	ss, err := Open(path, ds, hot, cold, cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -125,6 +139,10 @@ func testSplitStore(t *testing.T, cfg *Config) {
 	}

 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -259,8 +277,17 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 		t.Fatal(err)
 	}

+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
 	// open the splitstore
-	ss, err := Open("", ds, hot, cold, &Config{MarkSetType: "map"})
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -305,6 +332,10 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}

 	waitForCompaction := func() {
+		ss.txnSyncMx.Lock()
+		ss.txnSync = true
+		ss.txnSyncCond.Broadcast()
+		ss.txnSyncMx.Unlock()
 		for atomic.LoadInt32(&ss.compacting) == 1 {
 			time.Sleep(100 * time.Millisecond)
 		}
@@ -357,6 +388,235 @@ func TestSplitStoreSuppressCompactionNearUpgrade(t *testing.T) {
 	}
 }

+func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// first access using the standard view
+	err = f(context.Background(), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// nothing should be reified
+	waitForReification()
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+	// now make the hot/reifying view and ensure access reifies
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// everything should be reified
+	waitForReification()
+	for i, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !has {
+			t.Fatalf("block%d was not reified", i+1)
+		}
+	}
+}
+
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
+func TestSplitStoreReification(t *testing.T) {
+	t.Log("test reification with Has")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+	t.Log("test reification with Get")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Get(ctx, c)
+		return err
+	})
+	t.Log("test reification with GetSize")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.GetSize(ctx, c)
+		return err
+	})
+	t.Log("test reification with View")
+	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		return s.View(ctx, c, func(_ []byte) error { return nil })
+	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
+}
+
 type mockChain struct {
 	t testing.TB

@@ -426,17 +686,25 @@ func (c *mockChain) SubscribeHeadChanges(change func(revert []*types.TipSet, app

 type mockStore struct {
 	mx  sync.Mutex
-	set map[cid.Cid]blocks.Block
+	set map[string]blocks.Block
 }

 func newMockStore() *mockStore {
-	return &mockStore{set: make(map[cid.Cid]blocks.Block)}
+	return &mockStore{set: make(map[string]blocks.Block)}
+}
+
+func (b *mockStore) keyOf(c cid.Cid) string {
+	return string(c.Hash())
+}
+
+func (b *mockStore) cidOf(k string) cid.Cid {
+	return cid.NewCidV1(cid.Raw, mh.Multihash([]byte(k)))
 }

 func (b *mockStore) Has(_ context.Context, cid cid.Cid) (bool, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()
-	_, ok := b.set[cid]
+	_, ok := b.set[b.keyOf(cid)]
 	return ok, nil
 }

@@ -446,7 +714,7 @@ func (b *mockStore) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	blk, ok := b.set[cid]
+	blk, ok := b.set[b.keyOf(cid)]
 	if !ok {
 		return nil, blockstore.ErrNotFound
 	}
@@ -474,7 +742,7 @@ func (b *mockStore) Put(_ context.Context, blk blocks.Block) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	b.set[blk.Cid()] = blk
+	b.set[b.keyOf(blk.Cid())] = blk
 	return nil
 }

@@ -483,7 +751,7 @@ func (b *mockStore) PutMany(_ context.Context, blks []blocks.Block) error {
 	defer b.mx.Unlock()

 	for _, blk := range blks {
-		b.set[blk.Cid()] = blk
+		b.set[b.keyOf(blk.Cid())] = blk
 	}
 	return nil
 }
@@ -492,7 +760,7 @@ func (b *mockStore) DeleteBlock(_ context.Context, cid cid.Cid) error {
 	b.mx.Lock()
 	defer b.mx.Unlock()

-	delete(b.set, cid)
+	delete(b.set, b.keyOf(cid))
 	return nil
 }

@@ -501,7 +769,7 @@ func (b *mockStore) DeleteMany(_ context.Context, cids []cid.Cid) error {
 	defer b.mx.Unlock()

 	for _, c := range cids {
-		delete(b.set, c)
+		delete(b.set, b.keyOf(c))
 	}
 	return nil
 }
@@ -515,7 +783,7 @@ func (b *mockStore) ForEachKey(f func(cid.Cid) error) error {
 	defer b.mx.Unlock()

 	for c := range b.set {
-		err := f(c)
+		err := f(b.cidOf(c))
 		if err != nil {
 			return err
 		}
@@ -1,6 +1,7 @@
 package splitstore

 import (
+	"sync"
 	"sync/atomic"
 	"time"

@@ -55,12 +56,13 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 	if WarmupBoundary < epoch {
 		boundaryEpoch = epoch - WarmupBoundary
 	}
+	var mx sync.Mutex
 	batchHot := make([]blocks.Block, 0, batchSize)
-	count := int64(0)
-	xcount := int64(0)
-	missing := int64(0)
+	count := new(int64)
+	xcount := new(int64)
+	missing := new(int64)

-	visitor, err := s.markSetEnv.CreateVisitor("warmup", 0)
+	visitor, err := s.markSetEnv.New("warmup", 0)
 	if err != nil {
 		return xerrors.Errorf("error creating visitor: %w", err)
 	}
@@ -73,7 +75,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 				return errStopWalk
 			}

-			count++
+			atomic.AddInt64(count, 1)

 			has, err := s.hot.Has(s.ctx, c)
 			if err != nil {
@@ -87,22 +89,25 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 			blk, err := s.cold.Get(s.ctx, c)
 			if err != nil {
 				if err == bstore.ErrNotFound {
-					missing++
+					atomic.AddInt64(missing, 1)
 					return errStopWalk
 				}
 				return err
 			}

-			xcount++
+			atomic.AddInt64(xcount, 1)

+			mx.Lock()
 			batchHot = append(batchHot, blk)
 			if len(batchHot) == batchSize {
 				err = s.hot.PutMany(s.ctx, batchHot)
 				if err != nil {
+					mx.Unlock()
 					return err
 				}
 				batchHot = batchHot[:0]
 			}
+			mx.Unlock()

 			return nil
 		})
@@ -118,9 +123,9 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
 		}
 	}

-	log.Infow("warmup stats", "visited", count, "warm", xcount, "missing", missing)
+	log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing)

-	s.markSetSize = count + count>>2 // overestimate a bit
+	s.markSetSize = *count + *count>>2 // overestimate a bit
 	err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize))
 	if err != nil {
 		log.Warnf("error saving mark set size: %s", err)
@@ -1,6 +1,8 @@
 package splitstore

 import (
+	"sync"
+
 	cid "github.com/ipfs/go-cid"
 )

@@ -17,16 +19,42 @@ func (v *noopVisitor) Visit(_ cid.Cid) (bool, error) {
 	return true, nil
 }

-type cidSetVisitor struct {
+type tmpVisitor struct {
 	set *cid.Set
 }

-var _ ObjectVisitor = (*cidSetVisitor)(nil)
+var _ ObjectVisitor = (*tmpVisitor)(nil)
+
+func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}

-func (v *cidSetVisitor) Visit(c cid.Cid) (bool, error) {
 	return v.set.Visit(c), nil
 }

-func tmpVisitor() ObjectVisitor {
-	return &cidSetVisitor{set: cid.NewSet()}
+func newTmpVisitor() ObjectVisitor {
+	return &tmpVisitor{set: cid.NewSet()}
+}
+
+type concurrentVisitor struct {
+	mx  sync.Mutex
+	set *cid.Set
+}
+
+var _ ObjectVisitor = (*concurrentVisitor)(nil)
+
+func newConcurrentVisitor() *concurrentVisitor {
+	return &concurrentVisitor{set: cid.NewSet()}
+}
+
+func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
+	v.mx.Lock()
+	defer v.mx.Unlock()
+
+	return v.set.Visit(c), nil
 }
@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWBdRCBLUeKvoy22u5DcXs61adFn31v8WWCZgmBjDCjbsC
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWDUQJBA18njjXnG9RtLxoN3muvdU7PEy55QorUEsdAqdy
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWFHDtFx7CVTy4xoCDutVo1cScvSnQjDeaM8UzwVS1qwkh
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWKt8cwpkiumkT8x32c3YFxsPRwhV5J8hCYPn9mhUmcAXt
@@ -90,6 +90,7 @@ func init() {
 	UpgradeTurboHeight = getUpgradeHeight("LOTUS_ACTORSV4_HEIGHT", UpgradeTurboHeight)
 	UpgradeHyperdriveHeight = getUpgradeHeight("LOTUS_HYPERDRIVE_HEIGHT", UpgradeHyperdriveHeight)
 	UpgradeChocolateHeight = getUpgradeHeight("LOTUS_CHOCOLATE_HEIGHT", UpgradeChocolateHeight)
+	UpgradeOhSnapHeight = getUpgradeHeight("LOTUS_OHSNAP_HEIGHT", UpgradeOhSnapHeight)

 	BuildType |= Build2k

@@ -42,8 +42,7 @@ const UpgradeTurboHeight = -15
 const UpgradeHyperdriveHeight = -16
 const UpgradeChocolateHeight = -17

-// 2022-01-17T19:00:00Z
-const UpgradeOhSnapHeight = 30262
+const UpgradeOhSnapHeight = 240

 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2 << 30))
@@ -54,7 +54,8 @@ const UpgradeHyperdriveHeight = 420

 const UpgradeChocolateHeight = 312746

-const UpgradeOhSnapHeight = 99999999
+// 2022-02-10T19:23:00Z
+const UpgradeOhSnapHeight = 682006

 func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(32 << 30))
@@ -67,7 +67,8 @@ const UpgradeHyperdriveHeight = 892800
 // 2021-10-26T13:30:00Z
 const UpgradeChocolateHeight = 1231620

-var UpgradeOhSnapHeight = abi.ChainEpoch(999999999999)
+// 2022-03-01T15:00:00Z
+var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)

 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
@@ -13,6 +13,17 @@ import (

 func BlocksTopic(netName dtypes.NetworkName) string   { return "/fil/blocks/" + string(netName) }
 func MessagesTopic(netName dtypes.NetworkName) string { return "/fil/msgs/" + string(netName) }
+func IndexerIngestTopic(netName dtypes.NetworkName) string {
+
+	nn := string(netName)
+	// The network name testnetnet is here for historical reasons.
+	// Going forward we aim to use the name `mainnet` where possible.
+	if nn == "testnetnet" {
+		nn = "mainnet"
+	}
+
+	return "/indexer/ingest/" + nn
+}
 func DhtProtocolName(netName dtypes.NetworkName) protocol.ID {
 	return protocol.ID("/fil/kad/" + string(netName))
 }
@@ -30,23 +30,23 @@
    "sector_size": 2048
  },
  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.params": {
-    "cid": "QmeNqDvsvyam4vqwCkstbxgb9S7RZEUeBDrJvBWKcpFKr6",
-    "digest": "532b53883ed4f794cb9d0db583d0df59",
+    "cid": "QmNPc75iEfcahCwNKdqnWLtxnjspUGGR4iscjiz3wP3RtS",
+    "digest": "1b3cfd761a961543f9eb273e435a06a2",
    "sector_size": 34359738368
  },
  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-0-3b7f44a9362e3985369454947bc94022e118211e49fd672d52bec1cbfd599d18.vk": {
-    "cid": "QmdLWr6moLUPScJZwoBckWqAeJkrBPAJPNLz8mWAfTdmXH",
-    "digest": "46990eb1bf5159c394a10309f269c1b6",
+    "cid": "QmdFFUe1gcz9MMHc6YW8aoV48w4ckvcERjt7PkydQAMfCN",
+    "digest": "3a6941983754737fde880d29c7094905",
    "sector_size": 34359738368
  },
  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.params": {
-    "cid": "QmdQsi9uFhxK9cGwuK4rHuwKQoHkz6upYTCz4UdLiy1vA2",
-    "digest": "4223c63dbd94de1538006a14f37179e3",
+    "cid": "QmUB6xTVjzBQGuDNeyJMrrJ1byk58vhPm8eY2Lv9pgwanp",
+    "digest": "1a392e7b759fb18e036c7559b5ece816",
    "sector_size": 68719476736
  },
  "v28-empty-sector-update-merkletree-poseidon_hasher-8-8-2-102e1444a7e9a97ebf1e3d6855dcc77e66c011ea66f936d9b2c508f87f2f83a7.vk": {
-    "cid": "QmPirFX9wX99iMGA6zFY2CvcrdcDkj73X4MP6DLduvpbk9",
-    "digest": "ce39b614d788d3aef26bac1b28521d94",
+    "cid": "Qmd794Jty7k26XJ8Eg4NDEks65Qk8G4GVfGkwqvymv8HAg",
+    "digest": "80e366df2f1011953c2d01c7b7c9ee8e",
    "sector_size": 68719476736
  },
  "v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": {
@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }

 // BuildVersion is the local build version
-const BuildVersion = "1.14.0-rc3"
+const BuildVersion = "1.15.1-dev"

 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {
@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )

 func init() {
@@ -62,6 +63,11 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }

+type RemoveDataCapProposal = verifreg{{.latestVersion}}.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg{{.latestVersion}}.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg{{.latestVersion}}.RemoveDataCapParams
+type RmDcProposalID = verifreg{{.latestVersion}}.RmDcProposalID
+const SignatureDomainSeparation_RemoveDataCap = verifreg{{.latestVersion}}.SignatureDomainSeparation_RemoveDataCap

 type State interface {
 	cbor.Marshaler
@@ -69,6 +75,7 @@ type State interface {
 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}
@@ -61,6 +61,10 @@ func (s *state{{.v}}) VerifierDataCap(addr address.Address) (bool, abi.StoragePo
 	return getDataCap(s.store, actors.Version{{.v}}, s.verifiers, addr)
 }

+func (s *state{{.v}}) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version{{.v}}, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state{{.v}}) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version{{.v}}, s.verifiers, cb)
 }
@@ -77,6 +81,11 @@ func (s *state{{.v}}) verifiers() (adt.Map, error) {
 	return adt{{.v}}.AsMap(s.store, s.Verifiers{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
 }

+func (s *state{{.v}}) removeDataCapProposalIDs() (adt.Map, error) {
+    {{if le .v 6}}return nil, nil
+    {{else}}return adt{{.v}}.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin{{.v}}.DefaultHamtBitwidth){{end}}
+}
+
 func (s *state{{.v}}) GetState() interface{} {
 	return &s.State
 }
@@ -6,6 +6,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 	"golang.org/x/xerrors"
 )

@@ -50,3 +51,28 @@ func forEachCap(store adt.Store, ver actors.Version, root rootFunc, cb func(addr
 		return cb(a, dcap)
 	})
 }
+
+func getRemoveDataCapProposalID(store adt.Store, ver actors.Version, root rootFunc, verifier address.Address, client address.Address) (bool, uint64, error) {
+	if verifier.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	if client.Protocol() != address.ID {
+		return false, 0, xerrors.Errorf("can only look up ID addresses")
+	}
+	vh, err := root()
+	if err != nil {
+		return false, 0, xerrors.Errorf("loading verifreg: %w", err)
+	}
+	if vh == nil {
+		return false, 0, xerrors.Errorf("remove data cap proposal hamt not found. you are probably using an incompatible version of actors")
+	}
+
+	var id verifreg.RmDcProposalID
+	if found, err := vh.Get(abi.NewAddrPairKey(verifier, client), &id); err != nil {
+		return false, 0, xerrors.Errorf("looking up addr pair: %w", err)
+	} else if !found {
+		return false, 0, nil
+	}
+
+	return true, id.ProposalID, nil
+}
@@ -53,6 +53,10 @@ func (s *state0) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version0, s.verifiers, addr)
 }

+func (s *state0) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version0, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state0) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version0, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state0) verifiers() (adt.Map, error) {
 	return adt0.AsMap(s.store, s.Verifiers)
 }

+func (s *state0) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state0) GetState() interface{} {
 	return &s.State
 }
@@ -53,6 +53,10 @@ func (s *state2) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version2, s.verifiers, addr)
 }

+func (s *state2) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version2, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state2) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version2, s.verifiers, cb)
 }
@@ -69,6 +73,11 @@ func (s *state2) verifiers() (adt.Map, error) {
 	return adt2.AsMap(s.store, s.Verifiers)
 }

+func (s *state2) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state2) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state3) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version3, s.verifiers, addr)
 }

+func (s *state3) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version3, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state3) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version3, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state3) verifiers() (adt.Map, error) {
 	return adt3.AsMap(s.store, s.Verifiers, builtin3.DefaultHamtBitwidth)
 }

+func (s *state3) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state3) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state4) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version4, s.verifiers, addr)
 }

+func (s *state4) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version4, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state4) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version4, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state4) verifiers() (adt.Map, error) {
 	return adt4.AsMap(s.store, s.Verifiers, builtin4.DefaultHamtBitwidth)
 }

+func (s *state4) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state4) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state5) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version5, s.verifiers, addr)
 }

+func (s *state5) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version5, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state5) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version5, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state5) verifiers() (adt.Map, error) {
 	return adt5.AsMap(s.store, s.Verifiers, builtin5.DefaultHamtBitwidth)
 }

+func (s *state5) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state5) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state6) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version6, s.verifiers, addr)
 }

+func (s *state6) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version6, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state6) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version6, s.verifiers, cb)
 }
@@ -70,6 +74,11 @@ func (s *state6) verifiers() (adt.Map, error) {
 	return adt6.AsMap(s.store, s.Verifiers, builtin6.DefaultHamtBitwidth)
 }

+func (s *state6) removeDataCapProposalIDs() (adt.Map, error) {
+	return nil, nil
+
+}
+
 func (s *state6) GetState() interface{} {
 	return &s.State
 }
@@ -54,6 +54,10 @@ func (s *state7) VerifierDataCap(addr address.Address) (bool, abi.StoragePower,
 	return getDataCap(s.store, actors.Version7, s.verifiers, addr)
 }

+func (s *state7) RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error) {
+	return getRemoveDataCapProposalID(s.store, actors.Version7, s.removeDataCapProposalIDs, verifier, client)
+}
+
 func (s *state7) ForEachVerifier(cb func(addr address.Address, dcap abi.StoragePower) error) error {
 	return forEachCap(s.store, actors.Version7, s.verifiers, cb)
 }
@@ -70,6 +74,10 @@ func (s *state7) verifiers() (adt.Map, error) {
 	return adt7.AsMap(s.store, s.Verifiers, builtin7.DefaultHamtBitwidth)
 }

+func (s *state7) removeDataCapProposalIDs() (adt.Map, error) {
+	return adt7.AsMap(s.store, s.RemoveDataCapProposalIDs, builtin7.DefaultHamtBitwidth)
+}
+
 func (s *state7) GetState() interface{} {
 	return &s.State
 }
@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
+	verifreg7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg"
 )

 func init() {
@@ -151,12 +152,20 @@ func GetActorCodeID(av actors.Version) (cid.Cid, error) {
 	return cid.Undef, xerrors.Errorf("unknown actor version %d", av)
 }

+type RemoveDataCapProposal = verifreg7.RemoveDataCapProposal
+type RemoveDataCapRequest = verifreg7.RemoveDataCapRequest
+type RemoveDataCapParams = verifreg7.RemoveDataCapParams
+type RmDcProposalID = verifreg7.RmDcProposalID
+
+const SignatureDomainSeparation_RemoveDataCap = verifreg7.SignatureDomainSeparation_RemoveDataCap
+
 type State interface {
 	cbor.Marshaler

 	RootKey() (address.Address, error)
 	VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error)
 	VerifierDataCap(address.Address) (bool, abi.StoragePower, error)
+	RemoveDataCapProposalID(verifier address.Address, client address.Address) (bool, uint64, error)
 	ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error
 	ForEachClient(func(addr address.Address, dcap abi.StoragePower) error) error
 	GetState() interface{}
@@ -142,7 +142,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re

 	go func() {
 		start := build.Clock.Now()
-		log.Infow("start fetching randomness", "round", round)
+		log.Debugw("start fetching randomness", "round", round)
 		resp, err := db.client.Get(ctx, round)

 		var br beacon.Response
@@ -152,7 +152,7 @@ func (db *DrandBeacon) Entry(ctx context.Context, round uint64) <-chan beacon.Re
 			br.Entry.Round = resp.Round()
 			br.Entry.Data = resp.Signature()
 		}
-		log.Infow("done fetching randomness", "round", round, "took", build.Clock.Since(start))
+		log.Debugw("done fetching randomness", "round", round, "took", build.Clock.Since(start))
 		out <- br
 		close(out)
 	}()
@@ -32,6 +32,7 @@ import (

 	/* inline-gen end */

+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -92,16 +93,17 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 		partDone()
 	}()

-	makeVmWithBaseState := func(base cid.Cid) (*vm.VM, error) {
+	ctx = blockstore.WithHotView(ctx)
+	makeVmWithBaseStateAndEpoch := func(base cid.Cid, e abi.ChainEpoch) (*vm.VM, error) {
 		vmopt := &vm.VMOpts{
 			StateBase:      base,
-			Epoch:          epoch,
+			Epoch:          e,
 			Rand:           r,
 			Bstore:         sm.ChainStore().StateBlockstore(),
 			Actors:         NewActorRegistry(),
 			Syscalls:       sm.Syscalls,
 			CircSupplyCalc: sm.GetVMCirculatingSupply,
-			NtwkVersion:    sm.GetNtwkVersion,
+			NetworkVersion: sm.GetNetworkVersion(ctx, e),
 			BaseFee:        baseFee,
 			LookbackState:  stmgr.LookbackStateGetterForTipset(sm, ts),
 		}
@@ -109,12 +111,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 		return sm.VMConstructor()(ctx, vmopt)
 	}

-	vmi, err := makeVmWithBaseState(pstate)
-	if err != nil {
-		return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
-	}
-
-	runCron := func(epoch abi.ChainEpoch) error {
+	runCron := func(vmCron *vm.VM, epoch abi.ChainEpoch) error {
 		cronMsg := &types.Message{
 			To:         cron.Address,
 			From:       builtin.SystemActorAddr,
@@ -126,59 +123,58 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 			Method:     cron.Methods.EpochTick,
 			Params:     nil,
 		}
-		ret, err := vmi.ApplyImplicitMessage(ctx, cronMsg)
+		ret, err := vmCron.ApplyImplicitMessage(ctx, cronMsg)
 		if err != nil {
-			return err
+			return xerrors.Errorf("running cron: %w", err)
 		}
+
 		if em != nil {
 			if err := em.MessageApplied(ctx, ts, cronMsg.Cid(), cronMsg, ret, true); err != nil {
 				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
 		if ret.ExitCode != 0 {
-			return xerrors.Errorf("CheckProofSubmissions exit was non-zero: %d", ret.ExitCode)
+			return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
 		}

 		return nil
 	}

 	for i := parentEpoch; i < epoch; i++ {
+		var err error
 		if i > parentEpoch {
-			// run cron for null rounds if any
-			if err := runCron(i); err != nil {
-				return cid.Undef, cid.Undef, err
+			vmCron, err := makeVmWithBaseStateAndEpoch(pstate, i)
+			if err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("making cron vm: %w", err)
 			}

-			pstate, err = vmi.Flush(ctx)
+			// run cron for null rounds if any
+			if err = runCron(vmCron, i); err != nil {
+				return cid.Undef, cid.Undef, xerrors.Errorf("running cron: %w", err)
+			}
+
+			pstate, err = vmCron.Flush(ctx)
 			if err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("flushing vm: %w", err)
+				return cid.Undef, cid.Undef, xerrors.Errorf("flushing cron vm: %w", err)
 			}
 		}

 		// handle state forks
 		// XXX: The state tree
-		newState, err := sm.HandleStateForks(ctx, pstate, i, em, ts)
+		pstate, err = sm.HandleStateForks(ctx, pstate, i, em, ts)
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("error handling state forks: %w", err)
 		}
-
-		if pstate != newState {
-			vmi, err = makeVmWithBaseState(newState)
-			if err != nil {
-				return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
-			}
-		}
-
-		if err = vmi.SetBlockHeight(ctx, i+1); err != nil {
-			return cid.Undef, cid.Undef, xerrors.Errorf("error advancing vm an epoch: %w", err)
-		}
-
-		pstate = newState
 	}

 	partDone()
 	partDone = metrics.Timer(ctx, metrics.VMApplyMessages)

+	vmi, err := makeVmWithBaseStateAndEpoch(pstate, epoch)
+	if err != nil {
+		return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err)
+	}
+
 	var receipts []cbg.CBORMarshaler
 	processedMsgs := make(map[cid.Cid]struct{})
 	for _, b := range bms {
@@ -246,7 +242,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, sm *stmgr.StateManager
 	partDone()
 	partDone = metrics.Timer(ctx, metrics.VMApplyCron)

-	if err := runCron(epoch); err != nil {
+	if err := runCron(vmi, epoch); err != nil {
 		return cid.Cid{}, cid.Cid{}, err
 	}

@@ -302,7 +298,7 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManag
 		parentEpoch = parent.Height
 	}

-	r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon())
+	r := rand.NewStateRand(sm.ChainStore(), ts.Cids(), sm.Beacon(), sm.GetNetworkVersion)

 	blkmsgs, err := sm.ChainStore().BlockMsgsForTipset(ctx, ts)
 	if err != nil {
@@ -95,7 +95,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
 		return xerrors.Errorf("load parent tipset failed (%s): %w", h.Parents, err)
 	}

-	winPoStNv := filec.sm.GetNtwkVersion(ctx, baseTs.Height())
+	winPoStNv := filec.sm.GetNetworkVersion(ctx, baseTs.Height())

 	lbts, lbst, err := stmgr.GetLookbackTipSetForRound(ctx, filec.sm, baseTs, h.Height)
 	if err != nil {
@@ -182,7 +182,7 @@ func (filec *FilecoinEC) ValidateBlock(ctx context.Context, b *types.FullBlock)
 				}
 			}

-			return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
+			return xerrors.Errorf("parent state root did not match computed state (%s != %s)", h.ParentStateRoot, stateroot)
 		}

 		if precp != h.ParentMessageReceipts {
@@ -458,7 +458,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl

 	stateroot, _, err := filec.sm.TipSetState(ctx, baseTs)
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to compute tipsettate for %s: %w", baseTs.Key(), err)
 	}

 	st, err := state.LoadStateTree(filec.store.ActorStore(ctx), stateroot)
@@ -466,8 +466,8 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		return xerrors.Errorf("failed to load base state tree: %w", err)
 	}

-	nv := filec.sm.GetNtwkVersion(ctx, b.Header.Height)
-	pl := vm.PricelistByEpoch(baseTs.Height())
+	nv := filec.sm.GetNetworkVersion(ctx, b.Header.Height)
+	pl := vm.PricelistByEpoch(b.Header.Height)
 	var sumGasLimit int64
 	checkMsg := func(msg types.ChainMsg) error {
 		m := msg.VMMessage()
@@ -475,7 +475,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		// Phase 1: syntactic validation, as defined in the spec
 		minGas := pl.OnChainMessage(msg.ChainLength())
 		if err := m.ValidForBlockInclusion(minGas.Total(), nv); err != nil {
-			return err
+			return xerrors.Errorf("msg %s invalid for block inclusion: %w", m.Cid(), err)
 		}

 		// ValidForBlockInclusion checks if any single message does not exceed BlockGasLimit
@@ -488,10 +488,10 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		// Phase 2: (Partial) semantic validation:
 		// the sender exists and is an account actor, and the nonces make sense
 		var sender address.Address
-		if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version13 {
+		if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version13 {
 			sender, err = st.LookupID(m.From)
 			if err != nil {
-				return err
+				return xerrors.Errorf("failed to lookup sender %s: %w", m.From, err)
 			}
 		} else {
 			sender = m.From
@@ -541,7 +541,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl

 	smArr := blockadt.MakeEmptyArray(tmpstore)
 	for i, m := range b.SecpkMessages {
-		if filec.sm.GetNtwkVersion(ctx, b.Header.Height) >= network.Version14 {
+		if filec.sm.GetNetworkVersion(ctx, b.Header.Height) >= network.Version14 {
 			if m.Signature.Type != crypto.SigTypeSecp256k1 {
 				return xerrors.Errorf("block had invalid secpk message at index %d: %w", i, err)
 			}
@@ -574,12 +574,13 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl

 	bmroot, err := bmArr.Root()
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to root bls msgs: %w", err)
+
 	}

 	smroot, err := smArr.Root()
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to root secp msgs: %w", err)
 	}

 	mrcid, err := tmpstore.Put(ctx, &types.MsgMeta{
@@ -587,7 +588,7 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 		SecpkMessages: smroot,
 	})
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to put msg meta: %w", err)
 	}

 	if b.Header.Messages != mrcid {
@@ -595,7 +596,12 @@ func (filec *FilecoinEC) checkBlockMessages(ctx context.Context, b *types.FullBl
 	}

 	// Finally, flush.
-	return vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
+	err = vm.Copy(ctx, tmpbs, filec.store.ChainBlockstore(), mrcid)
+	if err != nil {
+		return xerrors.Errorf("failed to flush:%w", err)
+	}
+
+	return nil
 }

 func (filec *FilecoinEC) IsEpochBeyondCurrMax(epoch abi.ChainEpoch) bool {
@@ -165,13 +165,8 @@ func DefaultUpgradeSchedule() stmgr.UpgradeSchedule {
 		Migration: UpgradeActorsV7,
 		PreMigrations: []stmgr.PreMigration{{
 			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     120,
+			StartWithin:     180,
 			DontStartWithin: 60,
-			StopWithin:      35,
-		}, {
-			PreMigration:    PreUpgradeActorsV7,
-			StartWithin:     30,
-			DontStartWithin: 15,
 			StopWithin:      5,
 		}},
 		Expensive: true,
@@ -1264,7 +1259,7 @@ func upgradeActorsV7Common(
 	root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet,
 	config nv15.Config,
 ) (cid.Cid, error) {
-	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB)
+	writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB/4)
 	// TODO: pretty sure we'd achieve nothing by doing this, confirm in review
 	//buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), writeStore)
 	store := store.ActorStore(ctx, writeStore)
@@ -491,10 +491,8 @@ func VerifyPreSealedData(ctx context.Context, cs *store.ChainStore, sys vm.Sysca
 		Actors:         filcns.NewActorRegistry(),
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
-		NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
-			return nv
-		},
-		BaseFee: types.NewInt(0),
+		NetworkVersion: nv,
+		BaseFee:        types.NewInt(0),
 	}
 	vm, err := vm.NewVM(ctx, &vmopt)
 	if err != nil {
@@ -94,10 +94,8 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 		Actors:         filcns.NewActorRegistry(),
 		Syscalls:       mkFakedSigSyscalls(sys),
 		CircSupplyCalc: csc,
-		NtwkVersion: func(_ context.Context, _ abi.ChainEpoch) network.Version {
-			return nv
-		},
-		BaseFee: types.NewInt(0),
+		NetworkVersion: nv,
+		BaseFee:        types.NewInt(0),
 	}

 	vm, err := vm.NewVM(ctx, vmopt)
@@ -510,31 +508,13 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 // TODO: copied from actors test harness, deduplicate or remove from here
 type fakeRand struct{}

-func (fr *fakeRand) GetChainRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (fr *fakeRand) GetChainRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
 	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
 	return out, nil
 }

-func (fr *fakeRand) GetChainRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch * 1000))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomnessV3(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomnessV2(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	out := make([]byte, 32)
-	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
-	return out, nil
-}
-
-func (fr *fakeRand) GetBeaconRandomnessV1(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (fr *fakeRand) GetBeaconRandomness(ctx context.Context, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	out := make([]byte, 32)
 	_, _ = rand.New(rand.NewSource(int64(randEpoch))).Read(out) //nolint
 	return out, nil
@@ -106,7 +106,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 	curTs := mp.curTs
 	mp.curTsLk.Unlock()

-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1

 	var baseFee big.Int
 	if len(curTs.Blocks()) > 0 {
@@ -0,0 +1,224 @@
+//stm: #unit
+package messagepool
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
+	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
+)
+
+func init() {
+	_ = logging.SetLogLevel("*", "INFO")
+}
+
+func getCheckMessageStatus(statusCode api.CheckStatusCode, msgStatuses []api.MessageCheckStatus) (*api.MessageCheckStatus, error) {
+	for i := 0; i < len(msgStatuses); i++ {
+		iMsgStatuses := msgStatuses[i]
+		if iMsgStatuses.CheckStatus.Code == statusCode {
+			return &iMsgStatuses, nil
+		}
+	}
+	return nil, fmt.Errorf("Could not find CheckStatusCode %s", statusCode)
+}
+
+func TestCheckMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	var protos []*api.MessagePrototype
+	for i := 0; i < 5; i++ {
+		msg := &types.Message{
+			To:         target,
+			From:       sender,
+			Value:      types.NewInt(1),
+			Nonce:      uint64(i),
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 2<<10),
+		}
+		proto := &api.MessagePrototype{
+			Message:    *msg,
+			ValidNonce: true,
+		}
+		protos = append(protos, proto)
+	}
+
+	messageStatuses, err := mp.CheckMessages(context.TODO(), protos)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckPendingMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_PENDING_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	messageStatuses, err := mp.CheckPendingMessages(context.TODO(), sender)
+	assert.NoError(t, err)
+	for i := 0; i < len(messageStatuses); i++ {
+		iMsgStatuses := messageStatuses[i]
+		for j := 0; j < len(iMsgStatuses); j++ {
+			jStatus := iMsgStatuses[i]
+			assert.True(t, jStatus.OK)
+		}
+	}
+}
+
+func TestCheckReplaceMessages(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_CHECK_REPLACE_MESSAGES_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sender, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tma.setBalance(sender, 1000e15)
+	target := mock.Address(1001)
+
+	// add a valid message to the pool
+	msg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 2<<10),
+	}
+
+	sig, err := w.WalletSign(context.TODO(), sender, msg.Cid().Bytes(), api.MsgMeta{})
+	if err != nil {
+		panic(err)
+	}
+	sm := &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}
+	mustAdd(t, mp, sm)
+
+	// create a new message with the same data, except that it is too big
+	var msgs []*types.Message
+	invalidmsg := &types.Message{
+		To:         target,
+		From:       sender,
+		Value:      types.NewInt(1),
+		Nonce:      0,
+		GasLimit:   50000000,
+		GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+		GasPremium: types.NewInt(1),
+		Params:     make([]byte, 128<<10),
+	}
+	msgs = append(msgs, invalidmsg)
+
+	{
+		messageStatuses, err := mp.CheckReplaceMessages(context.TODO(), msgs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < len(messageStatuses); i++ {
+			iMsgStatuses := messageStatuses[i]
+
+			status, err := getCheckMessageStatus(api.CheckStatusMessageSize, iMsgStatuses)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// the replacement message should cause a status error
+			assert.False(t, status.OK)
+		}
+	}
+
+}
@@ -173,10 +173,17 @@ type MessagePool struct {

 	sigValCache *lru.TwoQueueCache

+	nonceCache *lru.Cache
+
 	evtTypes [3]journal.EventType
 	journal  journal.Journal
 }

+type nonceCacheKey struct {
+	tsk  types.TipSetKey
+	addr address.Address
+}
+
 type msgSet struct {
 	msgs          map[uint64]*types.SignedMessage
 	nextNonce     uint64
@@ -196,10 +203,10 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
 	return types.BigAdd(minPrice, types.NewInt(1))
 }

-func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSepc *api.MessageSendSpec) {
+func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.MessageSendSpec) {
 	var maxFee abi.TokenAmount
-	if sendSepc != nil {
-		maxFee = sendSepc.MaxFee
+	if sendSpec != nil {
+		maxFee = sendSpec.MaxFee
 	}
 	if maxFee.Int == nil || maxFee.Equals(big.Zero()) {
 		mf, err := mff()
@@ -361,6 +368,7 @@ func (ms *msgSet) toSlice() []*types.SignedMessage {
 func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.UpgradeSchedule, netName dtypes.NetworkName, j journal.Journal) (*MessagePool, error) {
 	cache, _ := lru.New2Q(build.BlsSignatureCacheSize)
 	verifcache, _ := lru.New2Q(build.VerifSigCacheSize)
+	noncecache, _ := lru.New(256)

 	cfg, err := loadConfig(ctx, ds)
 	if err != nil {
@@ -386,6 +394,7 @@ func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.Upgra
 		pruneCooldown:  make(chan struct{}, 1),
 		blsSigCache:    cache,
 		sigValCache:    verifcache,
+		nonceCache:     noncecache,
 		changes:        lps.New(50),
 		localMsgs:      namespace.Wrap(ds, datastore.NewKey(localMsgsDs)),
 		api:            api,
@@ -619,7 +628,7 @@ func (mp *MessagePool) addLocal(ctx context.Context, m *types.SignedMessage) err
 // For non local messages, if the message cannot be included in the next 20 blocks it returns
 // a (soft) validation error.
 func (mp *MessagePool) verifyMsgBeforeAdd(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) {
-	epoch := curTs.Height()
+	epoch := curTs.Height() + 1
 	minGas := vm.PricelistByEpoch(epoch).OnChainMessage(m.ChainLength())

 	if err := m.VMMessage().ValidForBlockInclusion(minGas.Total(), build.NewestNetworkVersion); err != nil {
@@ -1016,11 +1025,23 @@ func (mp *MessagePool) getStateNonce(ctx context.Context, addr address.Address,
 	done := metrics.Timer(ctx, metrics.MpoolGetNonceDuration)
 	defer done()

+	nk := nonceCacheKey{
+		tsk:  ts.Key(),
+		addr: addr,
+	}
+
+	n, ok := mp.nonceCache.Get(nk)
+	if ok {
+		return n.(uint64), nil
+	}
+
 	act, err := mp.api.GetActorAfter(addr, ts)
 	if err != nil {
 		return 0, err
 	}

+	mp.nonceCache.Add(nk, act.Nonce)
+
 	return act.Nonce, nil
 }

@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -8,6 +9,7 @@ import (

 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
@@ -206,6 +208,7 @@ func (tma *testMpoolAPI) ChainComputeBaseFee(ctx context.Context, ts *types.TipS

 func assertNonce(t *testing.T, mp *MessagePool, addr address.Address, val uint64) {
 	t.Helper()
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
 	n, err := mp.GetNonce(context.TODO(), addr, types.EmptyTSK)
 	if err != nil {
 		t.Fatal(err)
@@ -224,6 +227,8 @@ func mustAdd(t *testing.T, mp *MessagePool, msg *types.SignedMessage) {
 }

 func TestMessagePool(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_NONCE_001
+
 	tma := newTestMpoolAPI()

 	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
@@ -325,6 +330,7 @@ func TestCheckMessageBig(t *testing.T) {
 			Message:   *msg,
 			Signature: *sig,
 		}
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		err = mp.Add(context.TODO(), sm)
 		assert.ErrorIs(t, err, ErrMessageTooBig)
 	}
@@ -366,8 +372,10 @@ func TestMessagePoolMessagesInEachBlock(t *testing.T) {
 	tma.applyBlock(t, a)
 	tsa := mock.TipSet(a)

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	_, _ = mp.Pending(context.TODO())

+	//stm: @CHAIN_MEMPOOL_SELECT_001
 	selm, _ := mp.SelectMessages(context.Background(), tsa, 1)
 	if len(selm) == 0 {
 		t.Fatal("should have returned the rest of the messages")
@@ -428,6 +436,7 @@ func TestRevertMessages(t *testing.T) {

 	assertNonce(t, mp, sender, 4)

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	p, _ := mp.Pending(context.TODO())
 	fmt.Printf("%+v\n", p)
 	if len(p) != 3 {
@@ -486,6 +495,7 @@ func TestPruningSimple(t *testing.T) {

 	mp.Prune()

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	msgs, _ := mp.Pending(context.TODO())
 	if len(msgs) != 5 {
 		t.Fatal("expected only 5 messages in pool, got: ", len(msgs))
@@ -528,6 +538,7 @@ func TestLoadLocal(t *testing.T) {
 	msgs := make(map[cid.Cid]struct{})
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		cid, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -544,6 +555,7 @@ func TestLoadLocal(t *testing.T) {
 		t.Fatal(err)
 	}

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pmsgs, _ := mp.Pending(context.TODO())
 	if len(msgs) != len(pmsgs) {
 		t.Fatalf("expected %d messages, but got %d", len(msgs), len(pmsgs))
@@ -599,6 +611,7 @@ func TestClearAll(t *testing.T) {
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -610,8 +623,10 @@ func TestClearAll(t *testing.T) {
 		mustAdd(t, mp, m)
 	}

+	//stm: @CHAIN_MEMPOOL_CLEAR_001
 	mp.Clear(context.Background(), true)

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pending, _ := mp.Pending(context.TODO())
 	if len(pending) > 0 {
 		t.Fatalf("cleared the mpool, but got %d pending messages", len(pending))
@@ -654,6 +669,7 @@ func TestClearNonLocal(t *testing.T) {
 	gasLimit := gasguess.Costs[gasguess.CostKey{Code: builtin2.StorageMarketActorCodeID, M: 2}]
 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -665,8 +681,10 @@ func TestClearNonLocal(t *testing.T) {
 		mustAdd(t, mp, m)
 	}

+	//stm: @CHAIN_MEMPOOL_CLEAR_001
 	mp.Clear(context.Background(), false)

+	//stm: @CHAIN_MEMPOOL_PENDING_001
 	pending, _ := mp.Pending(context.TODO())
 	if len(pending) != 10 {
 		t.Fatalf("expected 10 pending messages, but got %d instead", len(pending))
@@ -724,6 +742,7 @@ func TestUpdates(t *testing.T) {

 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -745,3 +764,302 @@ func TestUpdates(t *testing.T) {
 		t.Fatal("expected closed channel, but got an update instead")
 	}
 }
+
+func TestMessageBelowMinGasFee(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	// fee is just below minimum gas fee
+	fee := minimumBaseFee.Uint64() - 1
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(fee),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.ErrorIs(t, err, ErrGasFeeCapTooLow)
+	}
+}
+
+func TestMessageValueTooHigh(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	totalFil := types.TotalFilecoinInt
+	extra := types.NewInt(1)
+
+	value := types.BigAdd(totalFil, extra)
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      value,
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
+		if err != nil {
+			panic(err)
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *sig,
+		}
+
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+	}
+}
+
+func TestMessageSignatureInvalid(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		msg := &types.Message{
+			To:         to,
+			From:       from,
+			Value:      types.NewInt(1),
+			Nonce:      0,
+			GasLimit:   50000000,
+			GasFeeCap:  types.NewInt(minimumBaseFee.Uint64()),
+			GasPremium: types.NewInt(1),
+			Params:     make([]byte, 32<<10),
+		}
+
+		badSig := &crypto.Signature{
+			Type: crypto.SigTypeSecp256k1,
+			Data: make([]byte, 0),
+		}
+		sm := &types.SignedMessage{
+			Message:   *msg,
+			Signature: *badSig,
+		}
+		err = mp.Add(context.TODO(), sm)
+		assert.Error(t, err)
+		// assert.Contains(t, err.Error(), "invalid signature length")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwice(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create a valid messages
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// try to add it twice
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "with nonce 0 already in mpool")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceNonceGap(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		// create message with invalid nonce (1)
+		sm := makeTestMessage(w, from, to, 1, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm)
+		// assert.Contains(t, err.Error(), "unfulfilled nonce gap")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiff(t *testing.T) {
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_001, minimumBaseFee.Uint64())
+
+		//stm: @CHAIN_MEMPOOL_PUSH_001
+		// then try to add message again
+		err = mp.Add(context.TODO(), sm2)
+		// assert.Contains(t, err.Error(), "replace by fee has too low GasPremium")
+		assert.Error(t, err)
+	}
+}
+
+func TestAddMessageTwiceCidDiffReplaced(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		// Create message with different data, so CID is different
+		sm2 := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64()*2)
+		mustAdd(t, mp, sm2)
+	}
+}
+
+func TestRemoveMessage(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_PUSH_001
+	tma := newTestMpoolAPI()
+
+	w, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	assert.NoError(t, err)
+
+	from, err := w.WalletNew(context.Background(), types.KTBLS)
+	assert.NoError(t, err)
+
+	tma.setBalance(from, 1000e9)
+
+	ds := datastore.NewMapDatastore()
+
+	mp, err := New(context.Background(), tma, ds, filcns.DefaultUpgradeSchedule(), "mptest", nil)
+	assert.NoError(t, err)
+
+	to := mock.Address(1001)
+
+	{
+		sm := makeTestMessage(w, from, to, 0, 50_000_000, minimumBaseFee.Uint64())
+		mustAdd(t, mp, sm)
+
+		//stm: @CHAIN_MEMPOOL_REMOVE_001
+		// remove message for sender
+		mp.Remove(context.TODO(), from, sm.Message.Nonce, true)
+
+		//stm: @CHAIN_MEMPOOL_PENDING_FOR_001
+		// check messages in pool: should be none present
+		msgs := mp.pendingFor(context.TODO(), from)
+		assert.Len(t, msgs, 0)
+	}
+}
@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -16,6 +17,7 @@ import (
 )

 func TestRepubMessages(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
 	oldRepublishBatchDelay := RepublishBatchDelay
 	RepublishBatchDelay = time.Microsecond
 	defer func() {
@@ -57,6 +59,7 @@ func TestRepubMessages(t *testing.T) {

 	for i := 0; i < 10; i++ {
 		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(i+1))
+		//stm: @CHAIN_MEMPOOL_PUSH_001
 		_, err := mp.Push(context.TODO(), m)
 		if err != nil {
 			t.Fatal(err)
@@ -1,3 +1,4 @@
+//stm: #unit
 package messagepool

 import (
@@ -74,6 +75,8 @@ func makeTestMpool() (*MessagePool, *testMpoolAPI) {
 }

 func TestMessageChains(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001
+	//stm: @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -310,6 +313,8 @@ func TestMessageChains(t *testing.T) {
 }

 func TestMessageChainSkipping(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_CREATE_MSG_CHAINS_001
+
 	// regression test for chain skip bug

 	mp, tma := makeTestMpool()
@@ -382,6 +387,7 @@ func TestMessageChainSkipping(t *testing.T) {
 }

 func TestBasicMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	oldMaxNonceGap := MaxNonceGap
 	MaxNonceGap = 1000
 	defer func() {
@@ -532,6 +538,7 @@ func TestBasicMessageSelection(t *testing.T) {
 }

 func TestMessageSelectionTrimmingGas(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -595,6 +602,7 @@ func TestMessageSelectionTrimmingGas(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -641,6 +649,7 @@ func TestMessageSelectionTrimmingMsgsBasic(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -707,6 +716,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersBasic(t *testing.T) {
 }

 func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -788,6 +798,7 @@ func TestMessageSelectionTrimmingMsgsTwoSendersAdvanced(t *testing.T) {
 }

 func TestPriorityMessageSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -867,6 +878,7 @@ func TestPriorityMessageSelection(t *testing.T) {
 }

 func TestPriorityMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -934,6 +946,7 @@ func TestPriorityMessageSelection2(t *testing.T) {
 }

 func TestPriorityMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
 	mp, tma := makeTestMpool()

 	// the actors
@@ -1028,6 +1041,8 @@ func TestPriorityMessageSelection3(t *testing.T) {
 }

 func TestOptimalMessageSelection1(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses just a single actor sending messages with a low tq
 	// the chain depenent merging algorithm should pick messages from the actor
 	// from the start
@@ -1094,6 +1109,8 @@ func TestOptimalMessageSelection1(t *testing.T) {
 }

 func TestOptimalMessageSelection2(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses two actors sending messages to each other, with the first
 	// actor paying (much) higher gas premium than the second.
 	// We select with a low ticket quality; the chain depenent merging algorithm should pick
@@ -1173,6 +1190,8 @@ func TestOptimalMessageSelection2(t *testing.T) {
 }

 func TestOptimalMessageSelection3(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// this test uses 10 actors sending a block of messages to each other, with the the first
 	// actors paying higher gas premium than the subsequent actors.
 	// We select with a low ticket quality; the chain dependent merging algorithm should pick
@@ -1416,6 +1435,8 @@ func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
 }

 func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1439,6 +1460,8 @@ func TestCompetitiveMessageSelectionExp(t *testing.T) {
 }

 func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @CHAIN_MEMPOOL_SELECT_001
+
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -1462,6 +1485,7 @@ func TestCompetitiveMessageSelectionZipf(t *testing.T) {
 }

 func TestGasReward(t *testing.T) {
+	//stm: @CHAIN_MEMPOOL_GET_GAS_REWARD_001
 	tests := []struct {
 		Premium   uint64
 		FeeCap    uint64
@@ -1494,6 +1518,8 @@ func TestGasReward(t *testing.T) {
 }

 func TestRealWorldSelection(t *testing.T) {
+	//stm: @TOKEN_WALLET_NEW_001, @TOKEN_WALLET_SIGN_001, @CHAIN_MEMPOOL_SELECT_001
+
 	// load test-messages.json.gz and rewrite the messages so that
 	// 1) we map each real actor to a test actor so that we can sign the messages
 	// 2) adjust the nonces so that they start from 0
@@ -1,3 +1,4 @@
+//stm: #unit
 package messagesigner

 import (
@@ -60,6 +61,7 @@ func TestMessageSignerSignMessage(t *testing.T) {
 	to2, err := w.WalletNew(ctx, types.KTSecp256k1)
 	require.NoError(t, err)

+	//stm: @CHAIN_MESSAGE_SIGNER_NEW_SIGNER_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_001, @CHAIN_MESSAGE_SIGNER_SIGN_MESSAGE_005
 	type msgSpec struct {
 		msg        *types.Message
 		mpoolNonce [1]uint64
@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/binary"

+	"github.com/filecoin-project/go-state-types/network"
+
 	logging "github.com/ipfs/go-log/v2"

 	"github.com/filecoin-project/lotus/chain/beacon"
@@ -70,7 +72,7 @@ func (sr *stateRand) GetBeaconRandomnessTipset(ctx context.Context, round abi.Ch
 	return randTs, nil
 }

-func (sr *stateRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte, lookback bool) ([]byte, error) {
+func (sr *stateRand) getChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte, lookback bool) ([]byte, error) {
 	_, span := trace.StartSpan(ctx, "store.GetChainRandomness")
 	defer span.End()
 	span.AddAttributes(trace.Int64Attribute("round", int64(round)))
@@ -101,32 +103,26 @@ func (sr *stateRand) GetChainRandomness(ctx context.Context, pers crypto.DomainS
 	return DrawRandomness(mtb.Ticket.VRFProof, pers, round, entropy)
 }

+type NetworkVersionGetter func(context.Context, abi.ChainEpoch) network.Version
+
 type stateRand struct {
-	cs     *store.ChainStore
-	blks   []cid.Cid
-	beacon beacon.Schedule
+	cs                   *store.ChainStore
+	blks                 []cid.Cid
+	beacon               beacon.Schedule
+	networkVersionGetter NetworkVersionGetter
 }

-func NewStateRand(cs *store.ChainStore, blks []cid.Cid, b beacon.Schedule) vm.Rand {
+func NewStateRand(cs *store.ChainStore, blks []cid.Cid, b beacon.Schedule, networkVersionGetter NetworkVersionGetter) vm.Rand {
 	return &stateRand{
-		cs:     cs,
-		blks:   blks,
-		beacon: b,
+		cs:                   cs,
+		blks:                 blks,
+		beacon:               b,
+		networkVersionGetter: networkVersionGetter,
 	}
 }

 // network v0-12
-func (sr *stateRand) GetChainRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	return sr.GetChainRandomness(ctx, pers, round, entropy, true)
-}
-
-// network v13 and on
-func (sr *stateRand) GetChainRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	return sr.GetChainRandomness(ctx, pers, round, entropy, false)
-}
-
-// network v0-12
-func (sr *stateRand) GetBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (sr *stateRand) getBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, true)
 	if err != nil {
 		return nil, err
@@ -143,7 +139,7 @@ func (sr *stateRand) GetBeaconRandomnessV1(ctx context.Context, pers crypto.Doma
 }

 // network v13
-func (sr *stateRand) GetBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (sr *stateRand) getBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	randTs, err := sr.GetBeaconRandomnessTipset(ctx, round, false)
 	if err != nil {
 		return nil, err
@@ -160,9 +156,9 @@ func (sr *stateRand) GetBeaconRandomnessV2(ctx context.Context, pers crypto.Doma
 }

 // network v14 and on
-func (sr *stateRand) GetBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+func (sr *stateRand) getBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
 	if filecoinEpoch < 0 {
-		return sr.GetBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
+		return sr.getBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
 	}

 	be, err := sr.extractBeaconEntryForEpoch(ctx, filecoinEpoch)
@@ -174,6 +170,28 @@ func (sr *stateRand) GetBeaconRandomnessV3(ctx context.Context, pers crypto.Doma
 	return DrawRandomness(be.Data, pers, filecoinEpoch, entropy)
 }

+func (sr *stateRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	nv := sr.networkVersionGetter(ctx, filecoinEpoch)
+
+	if nv >= network.Version13 {
+		return sr.getChainRandomness(ctx, pers, filecoinEpoch, entropy, false)
+	}
+
+	return sr.getChainRandomness(ctx, pers, filecoinEpoch, entropy, true)
+}
+
+func (sr *stateRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error) {
+	nv := sr.networkVersionGetter(ctx, filecoinEpoch)
+
+	if nv >= network.Version14 {
+		return sr.getBeaconRandomnessV3(ctx, pers, filecoinEpoch, entropy)
+	} else if nv == network.Version13 {
+		return sr.getBeaconRandomnessV2(ctx, pers, filecoinEpoch, entropy)
+	} else {
+		return sr.getBeaconRandomnessV1(ctx, pers, filecoinEpoch, entropy)
+	}
+}
+
 func (sr *stateRand) extractBeaconEntryForEpoch(ctx context.Context, filecoinEpoch abi.ChainEpoch) (*types.BeaconEntry, error) {
 	randTs, err := sr.GetBeaconRandomnessTipset(ctx, filecoinEpoch, false)
 	if err != nil {
@@ -1,3 +1,4 @@
+//stm:#unit
 package rand_test

 import (
@@ -55,11 +56,13 @@ func TestNullRandomnessV1(t *testing.T) {

 	randEpoch := ts.TipSet.TipSet().Height() - 2

+	//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V1_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01, @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_TIPSET_02
 	rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
 	if err != nil {
 		t.Fatal(err)
 	}

+	//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
 	bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(beforeNullHeight)+offset)

 	select {
@@ -68,6 +71,7 @@ func TestNullRandomnessV1(t *testing.T) {
 			t.Fatal(resp.Err)
 		}

+		//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01
 		rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
 		if err != nil {
 			t.Fatal(err)
@@ -131,11 +135,13 @@ func TestNullRandomnessV2(t *testing.T) {

 	randEpoch := ts.TipSet.TipSet().Height() - 2

+	//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V2_01
 	rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
 	if err != nil {
 		t.Fatal(err)
 	}

+	//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
 	bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(ts.TipSet.TipSet().Height())+offset)

 	select {
@@ -144,6 +150,7 @@ func TestNullRandomnessV2(t *testing.T) {
 			t.Fatal(resp.Err)
 		}

+		//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01, @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_TIPSET_03
 		// note that the randEpoch passed to DrawRandomness is still randEpoch (not the latest ts height)
 		rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
 		if err != nil {
@@ -212,11 +219,13 @@ func TestNullRandomnessV3(t *testing.T) {

 	randEpoch := ts.TipSet.TipSet().Height() - 2

+	//stm: @BLOCKCHAIN_RAND_GET_BEACON_RANDOMNESS_V3_01, @BLOCKCHAIN_RAND_EXTRACT_BEACON_ENTRY_FOR_EPOCH_01
 	rand1, err := cg.StateManager().GetRandomnessFromBeacon(ctx, pers, randEpoch, entropy, ts.TipSet.TipSet().Key())
 	if err != nil {
 		t.Fatal(err)
 	}

+	//stm: @BLOCKCHAIN_BEACON_GET_BEACON_FOR_EPOCH_01
 	bch := cg.BeaconSchedule().BeaconForEpoch(randEpoch).Entry(ctx, uint64(randEpoch)+offset)

 	select {
@@ -225,6 +234,7 @@ func TestNullRandomnessV3(t *testing.T) {
 			t.Fatal(resp.Err)
 		}

+		//stm: @BLOCKCHAIN_RAND_DRAW_RANDOMNESS_01
 		rand2, err := rand.DrawRandomness(resp.Entry.Data, pers, randEpoch, entropy)
 		if err != nil {
 			t.Fatal(err)
@@ -359,7 +359,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule
 		return nil, xerrors.Errorf("failed to get randomness for winning post: %w", err)
 	}

-	nv := sm.GetNtwkVersion(ctx, ts.Height())
+	nv := sm.GetNetworkVersion(ctx, ts.Height())

 	sectors, err := GetSectorsForWinningPoSt(ctx, nv, pv, sm, lbst, maddr, prand)
 	if err != nil {
@@ -421,7 +421,7 @@ func MinerEligibleToMine(ctx context.Context, sm *StateManager, addr address.Add
 	hmp, err := minerHasMinPower(ctx, sm, addr, lookbackTs)

 	// TODO: We're blurring the lines between a "runtime network version" and a "Lotus upgrade epoch", is that unavoidable?
-	if sm.GetNtwkVersion(ctx, baseTs.Height()) <= network.Version3 {
+	if sm.GetNetworkVersion(ctx, baseTs.Height()) <= network.Version3 {
 		return hmp, err
 	}

@@ -75,12 +75,12 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 	vmopt := &vm.VMOpts{
 		StateBase:      bstate,
 		Epoch:          pheight + 1,
-		Rand:           rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon),
+		Rand:           rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion),
 		Bstore:         sm.cs.StateBlockstore(),
 		Actors:         sm.tsExec.NewActorRegistry(),
 		Syscalls:       sm.Syscalls,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
-		NtwkVersion:    sm.GetNtwkVersion,
+		NetworkVersion: sm.GetNetworkVersion(ctx, pheight+1),
 		BaseFee:        types.NewInt(0),
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
@@ -186,7 +186,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		return nil, fmt.Errorf("failed to handle fork: %w", err)
 	}

-	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon)
+	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)

 	if span.IsRecordingEvents() {
 		span.AddAttributes(
@@ -204,7 +204,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		Actors:         sm.tsExec.NewActorRegistry(),
 		Syscalls:       sm.Syscalls,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
-		NtwkVersion:    sm.GetNtwkVersion,
+		NetworkVersion: sm.GetNetworkVersion(ctx, ts.Height()+1),
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
@@ -357,7 +357,7 @@ func (sm *StateManager) VMConstructor() func(context.Context, *vm.VMOpts) (*vm.V
 	}
 }

-func (sm *StateManager) GetNtwkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
+func (sm *StateManager) GetNetworkVersion(ctx context.Context, height abi.ChainEpoch) network.Version {
 	// The epochs here are the _last_ epoch for every version, or -1 if the
 	// version is disabled.
 	for _, spec := range sm.networkVersions {
@@ -378,16 +378,9 @@ func (sm *StateManager) GetRandomnessFromBeacon(ctx context.Context, personaliza
 		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
 	}

-	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon)
-	rnv := sm.GetNtwkVersion(ctx, randEpoch)
+	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)

-	if rnv >= network.Version14 {
-		return r.GetBeaconRandomnessV3(ctx, personalization, randEpoch, entropy)
-	} else if rnv == network.Version13 {
-		return r.GetBeaconRandomnessV2(ctx, personalization, randEpoch, entropy)
-	}
-
-	return r.GetBeaconRandomnessV1(ctx, personalization, randEpoch, entropy)
+	return r.GetBeaconRandomness(ctx, personalization, randEpoch, entropy)

 }

@@ -397,12 +390,7 @@ func (sm *StateManager) GetRandomnessFromTickets(ctx context.Context, personaliz
 		return nil, xerrors.Errorf("loading tipset key: %w", err)
 	}

-	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon)
-	rnv := sm.GetNtwkVersion(ctx, randEpoch)
+	r := rand.NewStateRand(sm.ChainStore(), pts.Cids(), sm.beacon, sm.GetNetworkVersion)

-	if rnv >= network.Version13 {
-		return r.GetChainRandomnessV2(ctx, personalization, randEpoch, entropy)
-	}
-
-	return r.GetChainRandomnessV1(ctx, personalization, randEpoch, entropy)
+	return r.GetChainRandomness(ctx, personalization, randEpoch, entropy)
 }
@@ -79,7 +79,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		// future. It's not guaranteed to be accurate... but that's fine.
 	}

-	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon)
+	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)
 	vmopt := &vm.VMOpts{
 		StateBase:      base,
 		Epoch:          height,
@@ -88,7 +88,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		Actors:         sm.tsExec.NewActorRegistry(),
 		Syscalls:       sm.Syscalls,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
-		NtwkVersion:    sm.GetNtwkVersion,
+		NetworkVersion: sm.GetNetworkVersion(ctx, height),
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 	}
@@ -128,7 +128,7 @@ func LookbackStateGetterForTipset(sm *StateManager, ts *types.TipSet) vm.Lookbac

 func GetLookbackTipSetForRound(ctx context.Context, sm *StateManager, ts *types.TipSet, round abi.ChainEpoch) (*types.TipSet, cid.Cid, error) {
 	var lbr abi.ChainEpoch
-	lb := policy.GetWinningPoStSectorSetLookback(sm.GetNtwkVersion(ctx, round))
+	lb := policy.GetWinningPoStSectorSetLookback(sm.GetNetworkVersion(ctx, round))
 	if round > lb {
 		lbr = round - lb
 	}
@@ -1,19 +1,24 @@
 package sub

 import (
+	"bytes"
 	"context"
-	"fmt"
+	"encoding/binary"
+	"sync"
 	"time"

 	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/sub/ratelimit"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node/impl/client"
+	"github.com/filecoin-project/lotus/node/impl/full"
 	lru "github.com/hashicorp/golang-lru"
 	blocks "github.com/ipfs/go-block-format"
 	bserv "github.com/ipfs/go-blockservice"
@@ -168,12 +173,12 @@ func fetchCids(
 	cidIndex := make(map[cid.Cid]int)
 	for i, c := range cids {
 		if c.Prefix() != msgCidPrefix {
-			return fmt.Errorf("invalid msg CID: %s", c)
+			return xerrors.Errorf("invalid msg CID: %s", c)
 		}
 		cidIndex[c] = i
 	}
 	if len(cids) != len(cidIndex) {
-		return fmt.Errorf("duplicate CIDs in fetchCids input")
+		return xerrors.Errorf("duplicate CIDs in fetchCids input")
 	}

 	for block := range bserv.GetBlocks(ctx, cids) {
@@ -196,7 +201,7 @@ func fetchCids(
 	if len(cidIndex) > 0 {
 		err := ctx.Err()
 		if err == nil {
-			err = fmt.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
+			err = xerrors.Errorf("failed to fetch %d messages for unknown reasons", len(cidIndex))
 		}
 		return err
 	}
@@ -444,3 +449,168 @@ func recordFailure(ctx context.Context, metric *stats.Int64Measure, failureType
 	)
 	stats.Record(ctx, metric.M(1))
 }
+
+type peerMsgInfo struct {
+	peerID    peer.ID
+	lastCid   cid.Cid
+	lastSeqno uint64
+	rateLimit *ratelimit.Window
+	mutex     sync.Mutex
+}
+
+type IndexerMessageValidator struct {
+	self peer.ID
+
+	peerCache *lru.TwoQueueCache
+	chainApi  full.ChainModuleAPI
+	stateApi  full.StateModuleAPI
+}
+
+func NewIndexerMessageValidator(self peer.ID, chainApi full.ChainModuleAPI, stateApi full.StateModuleAPI) *IndexerMessageValidator {
+	peerCache, _ := lru.New2Q(8192)
+
+	return &IndexerMessageValidator{
+		self:      self,
+		peerCache: peerCache,
+		chainApi:  chainApi,
+		stateApi:  stateApi,
+	}
+}
+
+func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubsub.Message) pubsub.ValidationResult {
+	// This chain-node should not be publishing its own messages.  These are
+	// relayed from market-nodes.  If a node appears to be local, reject it.
+	if pid == v.self {
+		log.Debug("ignoring indexer message from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+	originPeer := msg.GetFrom()
+	if originPeer == v.self {
+		log.Debug("ignoring indexer message originating from self")
+		stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+		return pubsub.ValidationIgnore
+	}
+
+	idxrMsg := dtsync.Message{}
+	err := idxrMsg.UnmarshalCBOR(bytes.NewBuffer(msg.Data))
+	if err != nil {
+		log.Errorw("Could not decode indexer pubsub message", "err", err)
+		return pubsub.ValidationReject
+	}
+	if len(idxrMsg.ExtraData) == 0 {
+		log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
+		return pubsub.ValidationIgnore
+	}
+
+	// Get miner info from lotus
+	minerAddr, err := address.NewFromBytes(idxrMsg.ExtraData)
+	if err != nil {
+		log.Warnw("cannot parse extra data as miner address", "err", err, "extraData", idxrMsg.ExtraData)
+		return pubsub.ValidationReject
+	}
+
+	minerID := minerAddr.String()
+	msgCid := idxrMsg.Cid
+
+	var msgInfo *peerMsgInfo
+	val, ok := v.peerCache.Get(minerID)
+	if !ok {
+		msgInfo = &peerMsgInfo{}
+	} else {
+		msgInfo = val.(*peerMsgInfo)
+	}
+
+	// Lock this peer's message info.
+	msgInfo.mutex.Lock()
+	defer msgInfo.mutex.Unlock()
+
+	if ok {
+		// Reject replayed messages.
+		seqno := binary.BigEndian.Uint64(msg.Message.GetSeqno())
+		if seqno <= msgInfo.lastSeqno {
+			log.Debugf("ignoring replayed indexer message")
+			return pubsub.ValidationIgnore
+		}
+		msgInfo.lastSeqno = seqno
+	}
+
+	if !ok || originPeer != msgInfo.peerID {
+		// Check that the miner ID maps to the peer that sent the message.
+		err = v.authenticateMessage(ctx, minerAddr, originPeer)
+		if err != nil {
+			log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerID)
+			stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
+			return pubsub.ValidationReject
+		}
+		msgInfo.peerID = originPeer
+		if !ok {
+			// Add msgInfo to cache only after being authenticated.  If two
+			// messages from the same peer are handled concurrently, there is a
+			// small chance that one msgInfo could replace the other here when
+			// the info is first cached.  This is OK, so no need to prevent it.
+			v.peerCache.Add(minerID, msgInfo)
+		}
+	}
+
+	// See if message needs to be ignored due to rate limiting.
+	if v.rateLimitPeer(msgInfo, msgCid) {
+		return pubsub.ValidationIgnore
+	}
+
+	stats.Record(ctx, metrics.IndexerMessageValidationSuccess.M(1))
+	return pubsub.ValidationAccept
+}
+
+func (v *IndexerMessageValidator) rateLimitPeer(msgInfo *peerMsgInfo, msgCid cid.Cid) bool {
+	const (
+		msgLimit        = 5
+		msgTimeLimit    = 10 * time.Second
+		repeatTimeLimit = 2 * time.Hour
+	)
+
+	timeWindow := msgInfo.rateLimit
+
+	// Check overall message rate.
+	if timeWindow == nil {
+		timeWindow = ratelimit.NewWindow(msgLimit, msgTimeLimit)
+		msgInfo.rateLimit = timeWindow
+	} else if msgInfo.lastCid == msgCid {
+		// Check if this is a repeat of the previous message data.
+		if time.Since(timeWindow.Newest()) < repeatTimeLimit {
+			log.Warnw("ignoring repeated indexer message", "sender", msgInfo.peerID)
+			return true
+		}
+	}
+
+	err := timeWindow.Add()
+	if err != nil {
+		log.Warnw("ignoring indexer message", "sender", msgInfo.peerID, "err", err)
+		return true
+	}
+
+	msgInfo.lastCid = msgCid
+
+	return false
+}
+
+func (v *IndexerMessageValidator) authenticateMessage(ctx context.Context, minerAddress address.Address, peerID peer.ID) error {
+	ts, err := v.chainApi.ChainHead(ctx)
+	if err != nil {
+		return err
+	}
+
+	minerInfo, err := v.stateApi.StateMinerInfo(ctx, minerAddress, ts.Key())
+	if err != nil {
+		return err
+	}
+
+	if minerInfo.PeerId == nil {
+		return xerrors.New("no peer id for miner")
+	}
+	if *minerInfo.PeerId != peerID {
+		return xerrors.New("miner id does not map to peer that sent message")
+	}
+
+	return nil
+}
@@ -1,13 +1,21 @@
+//stm: #unit
 package sub

 import (
+	"bytes"
 	"context"
 	"testing"

-	address "github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-legs/dtsync"
+	"github.com/filecoin-project/lotus/api/mocks"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/golang/mock/gomock"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	pb "github.com/libp2p/go-libp2p-pubsub/pb"
 )

 type getter struct {
@@ -49,6 +57,7 @@ func TestFetchCidsWithDedup(t *testing.T) {
 	}
 	g := &getter{msgs}

+	//stm: @CHAIN_INCOMING_FETCH_MESSAGES_BY_CID_001
 	// the cids have a duplicate
 	res, err := FetchMessagesByCids(context.TODO(), g, append(cids, cids[0]))

@@ -61,3 +70,65 @@ func TestFetchCidsWithDedup(t *testing.T) {
 		t.Fatalf("there is a nil message: first %p, last %p", res[0], res[len(res)-1])
 	}
 }
+
+func TestIndexerMessageValidator_Validate(t *testing.T) {
+	validCid, err := cid.Decode("QmbpDgg5kRLDgMxS8vPKNFXEcA6D5MC4CkuUdSWDVtHPGK")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		name           string
+		selfPID        string
+		senderPID      string
+		extraData      []byte
+		wantValidation pubsub.ValidationResult
+	}{
+		{
+			name:           "invalid extra data is rejected",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWE8yt84RVwW3sFcd6WMjbUdWrZer2YtT4dmtj3dHdahSZ",
+			extraData:      []byte("f0127896"), // note, casting encoded address to byte is invalid.
+			wantValidation: pubsub.ValidationReject,
+		},
+		{
+			name:           "same sender and receiver is ignored",
+			selfPID:        "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			senderPID:      "12D3KooWQiCbqEStCkdqUvr69gQsrp9urYJZUCkzsQXia7mbqbFW",
+			wantValidation: pubsub.ValidationIgnore,
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			mc := gomock.NewController(t)
+			node := mocks.NewMockFullNode(mc)
+			subject := NewIndexerMessageValidator(peer.ID(tc.selfPID), node, node)
+			message := dtsync.Message{
+				Cid:       validCid,
+				Addrs:     nil,
+				ExtraData: tc.extraData,
+			}
+			buf := bytes.NewBuffer(nil)
+			if err := message.MarshalCBOR(buf); err != nil {
+				t.Fatal(err)
+			}
+
+			topic := "topic"
+			pbm := &pb.Message{
+				Data:  buf.Bytes(),
+				Topic: &topic,
+				From:  nil,
+				Seqno: nil,
+			}
+			validate := subject.Validate(context.Background(), peer.ID(tc.senderPID), &pubsub.Message{
+				Message:       pbm,
+				ReceivedFrom:  peer.ID(tc.senderPID),
+				ValidatorData: nil,
+			})
+
+			if validate != tc.wantValidation {
+				t.Fatalf("expected %v but got %v", tc.wantValidation, validate)
+			}
+		})
+	}
+}
@@ -0,0 +1,89 @@
+package ratelimit
+
+import "errors"
+
+var ErrRateLimitExceeded = errors.New("rate limit exceeded")
+
+type queue struct {
+	buf   []int64
+	count int
+	head  int
+	tail  int
+}
+
+// cap returns the queue capacity
+func (q *queue) cap() int {
+	return len(q.buf)
+}
+
+// len returns the number of items in the queue
+func (q *queue) len() int {
+	return q.count
+}
+
+// push adds an element to the end of the queue.
+func (q *queue) push(elem int64) error {
+	if q.count == len(q.buf) {
+		return ErrRateLimitExceeded
+	}
+
+	q.buf[q.tail] = elem
+	// Calculate new tail position.
+	q.tail = q.next(q.tail)
+	q.count++
+	return nil
+}
+
+// pop removes and returns the element from the front of the queue.
+func (q *queue) pop() int64 {
+	if q.count <= 0 {
+		panic("pop from empty queue")
+	}
+	ret := q.buf[q.head]
+
+	// Calculate new head position.
+	q.head = q.next(q.head)
+	q.count--
+
+	return ret
+}
+
+// front returns the element at the front of the queue.  This is the element
+// that would be returned by pop().  This call panics if the queue is empty.
+func (q *queue) front() int64 {
+	if q.count <= 0 {
+		panic("front() called when empty")
+	}
+	return q.buf[q.head]
+}
+
+// back returns the element at the back of the queue.  This call panics if the
+// queue is empty.
+func (q *queue) back() int64 {
+	if q.count <= 0 {
+		panic("back() called when empty")
+	}
+	return q.buf[q.prev(q.tail)]
+}
+
+// prev returns the previous buffer position wrapping around buffer.
+func (q *queue) prev(i int) int {
+	if i == 0 {
+		return len(q.buf) - 1
+	}
+	return (i - 1) % len(q.buf)
+}
+
+// next returns the next buffer position wrapping around buffer.
+func (q *queue) next(i int) int {
+	return (i + 1) % len(q.buf)
+}
+
+// truncate pops values that are less than or equal the specified threshold.
+func (q *queue) truncate(threshold int64) {
+	for q.count != 0 && q.buf[q.head] <= threshold {
+		// pop() without returning a value
+		q.head = q.next(q.head)
+		q.count--
+	}
+}
@@ -0,0 +1,70 @@
+package ratelimit
+
+import "time"
+
+// Window is a time windows for counting events within a span of time.  The
+// windows slides forward in time so that it spans from the most recent event
+// to size time in the past.
+type Window struct {
+	q    *queue
+	size int64
+}
+
+// NewWindow creates a new Window that limits the number of events to maximum
+// count of events within a duration of time.  The capacity sets the maximum
+// number of events, and size sets the span of time over which the events are
+// counted.
+func NewWindow(capacity int, size time.Duration) *Window {
+	return &Window{
+		q: &queue{
+			buf: make([]int64, capacity),
+		},
+		size: int64(size),
+	}
+}
+
+// Add attempts to append a new timestamp into the current window.  Previously
+// added values that are not not within `size` difference from the value being
+// added are first removed.  Add fails if adding the value would cause the
+// window to exceed capacity.
+func (w *Window) Add() error {
+	now := time.Now().UnixNano()
+	if w.Len() != 0 {
+		w.q.truncate(now - w.size)
+	}
+	return w.q.push(now)
+}
+
+// Cap returns the maximum number of items the window can hold.
+func (w *Window) Cap() int {
+	return w.q.cap()
+}
+
+// Len returns the number of elements currently in the window.
+func (w *Window) Len() int {
+	return w.q.len()
+}
+
+// Span returns the distance from the first to the last item in the window.
+func (w *Window) Span() time.Duration {
+	if w.q.len() < 2 {
+		return 0
+	}
+	return time.Duration(w.q.back() - w.q.front())
+}
+
+// Oldest returns the oldest timestamp in the window.
+func (w *Window) Oldest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.front())
+}
+
+// Newest returns the newest timestamp in the window.
+func (w *Window) Newest() time.Time {
+	if w.q.len() == 0 {
+		return time.Time{}
+	}
+	return time.Unix(0, w.q.back())
+}
@@ -0,0 +1,61 @@
+package ratelimit
+
+import (
+	"testing"
+	"time"
+)
+
+func TestWindow(t *testing.T) {
+	const (
+		maxEvents = 3
+		timeLimit = 100 * time.Millisecond
+	)
+	w := NewWindow(maxEvents, timeLimit)
+	if w.Len() != 0 {
+		t.Fatal("q.Len() =", w.Len(), "expect 0")
+	}
+	if w.Cap() != maxEvents {
+		t.Fatal("q.Cap() =", w.Cap(), "expect 3")
+	}
+	if !w.Newest().IsZero() {
+		t.Fatal("expected newest to be zero time with empty window")
+	}
+	if !w.Oldest().IsZero() {
+		t.Fatal("expected oldest to be zero time with empty window")
+	}
+	if w.Span() != 0 {
+		t.Fatal("expected span to be zero time with empty window")
+	}
+
+	var err error
+	for i := 0; i < maxEvents; i++ {
+		err = w.Add()
+		if err != nil {
+			t.Fatalf("cannot add event %d", i)
+		}
+	}
+	if w.Len() != maxEvents {
+		t.Fatalf("q.Len() is %d, expected %d", w.Len(), maxEvents)
+	}
+	if err = w.Add(); err != ErrRateLimitExceeded {
+		t.Fatalf("add event %d within time limit should have failed with err: %s", maxEvents+1, ErrRateLimitExceeded)
+	}
+
+	time.Sleep(timeLimit)
+	if err = w.Add(); err != nil {
+		t.Fatalf("cannot add event after time limit: %s", err)
+	}
+
+	prev := w.Newest()
+	time.Sleep(timeLimit)
+	err = w.Add()
+	if err != nil {
+		t.Fatalf("cannot add event")
+	}
+	if w.Newest().Before(prev) {
+		t.Fatal("newest is before previous value")
+	}
+	if w.Oldest().Before(prev) {
+		t.Fatal("oldest is before previous value")
+	}
+}
@@ -1,3 +1,4 @@
+//stm: #unit
 package chain_test

 import (
@@ -103,7 +104,7 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil {
 		ctx:    ctx,
 		cancel: cancel,

-		mn: mocknet.New(ctx),
+		mn: mocknet.New(),
 		g:  g,
 		us: filcns.DefaultUpgradeSchedule(),
 	}
@@ -157,7 +158,7 @@ func prepSyncTestWithV5Height(t testing.TB, h int, v5height abi.ChainEpoch) *syn
 		ctx:    ctx,
 		cancel: cancel,

-		mn: mocknet.New(ctx),
+		mn: mocknet.New(),
 		g:  g,
 		us: sched,
 	}
@@ -462,6 +463,8 @@ func (tu *syncTestUtil) waitUntilSyncTarget(to int, target *types.TipSet) {
 }

 func TestSyncSimple(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 50
 	tu := prepSyncTest(t, H)

@@ -478,6 +481,8 @@ func TestSyncSimple(t *testing.T) {
 }

 func TestSyncMining(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 50
 	tu := prepSyncTest(t, H)

@@ -500,6 +505,8 @@ func TestSyncMining(t *testing.T) {
 }

 func TestSyncBadTimestamp(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 50
 	tu := prepSyncTest(t, H)

@@ -554,6 +561,8 @@ func (wpp badWpp) ComputeProof(context.Context, []proof7.ExtendedSectorInfo, abi
 }

 func TestSyncBadWinningPoSt(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 15
 	tu := prepSyncTest(t, H)

@@ -583,6 +592,9 @@ func (tu *syncTestUtil) loadChainToNode(to int) {
 }

 func TestSyncFork(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -650,6 +662,9 @@ func TestSyncFork(t *testing.T) {
 // A and B both include _different_ messages from sender X with nonce N (where N is the correct nonce for X).
 // We can confirm that the state can be correctly computed, and that `MessagesForTipset` behaves as expected.
 func TestDuplicateNonce(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001
+	//stm: @CHAIN_SYNCER_NEW_PEER_HEAD_001, @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -704,6 +719,7 @@ func TestDuplicateNonce(t *testing.T) {

 	var includedMsg cid.Cid
 	var skippedMsg cid.Cid
+	//stm: @CHAIN_STATE_SEARCH_MSG_001
 	r0, err0 := tu.nds[0].StateSearchMsg(context.TODO(), ts2.TipSet().Key(), msgs[0][0].Cid(), api.LookbackNoLimit, true)
 	r1, err1 := tu.nds[0].StateSearchMsg(context.TODO(), ts2.TipSet().Key(), msgs[1][0].Cid(), api.LookbackNoLimit, true)

@@ -745,6 +761,9 @@ func TestDuplicateNonce(t *testing.T) {
 // This test asserts that a block that includes a message with bad nonce can't be synced. A nonce is "bad" if it can't
 // be applied on the parent state.
 func TestBadNonce(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -792,6 +811,9 @@ func TestBadNonce(t *testing.T) {
 // One of the messages uses the sender's robust address, the other uses the ID address.
 // Such a block is invalid and should not sync.
 func TestMismatchedNoncesRobustID(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
 	v5h := abi.ChainEpoch(4)
 	tu := prepSyncTestWithV5Height(t, int(v5h+5), v5h)

@@ -804,6 +826,7 @@ func TestMismatchedNoncesRobustID(t *testing.T) {
 	require.NoError(t, err)

 	// Produce a message from the banker
+	//stm: @CHAIN_STATE_LOOKUP_ID_001
 	makeMsg := func(id bool) *types.SignedMessage {
 		sender := tu.g.Banker()
 		if id {
@@ -846,6 +869,9 @@ func TestMismatchedNoncesRobustID(t *testing.T) {
 // One of the messages uses the sender's robust address, the other uses the ID address.
 // Such a block is valid and should sync.
 func TestMatchedNoncesRobustID(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
 	v5h := abi.ChainEpoch(4)
 	tu := prepSyncTestWithV5Height(t, int(v5h+5), v5h)

@@ -858,6 +884,7 @@ func TestMatchedNoncesRobustID(t *testing.T) {
 	require.NoError(t, err)

 	// Produce a message from the banker with specified nonce
+	//stm: @CHAIN_STATE_LOOKUP_ID_001
 	makeMsg := func(n uint64, id bool) *types.SignedMessage {
 		sender := tu.g.Banker()
 		if id {
@@ -917,6 +944,8 @@ func runSyncBenchLength(b *testing.B, l int) {
 }

 func TestSyncInputs(t *testing.T) {
+	//stm: @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_VALIDATE_BLOCK_001,
+	//stm: @CHAIN_SYNCER_START_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -944,6 +973,9 @@ func TestSyncInputs(t *testing.T) {
 }

 func TestSyncCheckpointHead(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -963,6 +995,7 @@ func TestSyncCheckpointHead(t *testing.T) {
 	a = tu.mineOnBlock(a, p1, []int{0}, true, false, nil, 0, true)

 	tu.waitUntilSyncTarget(p1, a.TipSet())
+	//stm: @CHAIN_SYNCER_CHECKPOINT_001
 	tu.checkpointTs(p1, a.TipSet().Key())

 	require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
@@ -982,15 +1015,20 @@ func TestSyncCheckpointHead(t *testing.T) {
 	tu.waitUntilNodeHasTs(p1, b.TipSet().Key())
 	p1Head := tu.getHead(p1)
 	require.True(tu.t, p1Head.Equals(a.TipSet()))
+	//stm: @CHAIN_SYNCER_CHECK_BAD_001
 	tu.assertBad(p1, b.TipSet())

 	// Should be able to switch forks.
+	//stm: @CHAIN_SYNCER_CHECKPOINT_001
 	tu.checkpointTs(p1, b.TipSet().Key())
 	p1Head = tu.getHead(p1)
 	require.True(tu.t, p1Head.Equals(b.TipSet()))
 }

 func TestSyncCheckpointEarlierThanHead(t *testing.T) {
+	//stm: @BLOCKCHAIN_BEACON_VALIDATE_BLOCK_VALUES_01, @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_SYNC_001, @CHAIN_SYNCER_COLLECT_CHAIN_001, @CHAIN_SYNCER_COLLECT_HEADERS_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_VALIDATE_TIPSET_001, @CHAIN_SYNCER_STOP_001
 	H := 10
 	tu := prepSyncTest(t, H)

@@ -1010,6 +1048,7 @@ func TestSyncCheckpointEarlierThanHead(t *testing.T) {
 	a = tu.mineOnBlock(a, p1, []int{0}, true, false, nil, 0, true)

 	tu.waitUntilSyncTarget(p1, a.TipSet())
+	//stm: @CHAIN_SYNCER_CHECKPOINT_001
 	tu.checkpointTs(p1, a1.TipSet().Key())

 	require.NoError(t, tu.g.ResyncBankerNonce(a1.TipSet()))
@@ -1029,15 +1068,19 @@ func TestSyncCheckpointEarlierThanHead(t *testing.T) {
 	tu.waitUntilNodeHasTs(p1, b.TipSet().Key())
 	p1Head := tu.getHead(p1)
 	require.True(tu.t, p1Head.Equals(a.TipSet()))
+	//stm: @CHAIN_SYNCER_CHECK_BAD_001
 	tu.assertBad(p1, b.TipSet())

 	// Should be able to switch forks.
+	//stm: @CHAIN_SYNCER_CHECKPOINT_001
 	tu.checkpointTs(p1, b.TipSet().Key())
 	p1Head = tu.getHead(p1)
 	require.True(tu.t, p1Head.Equals(b.TipSet()))
 }

 func TestInvalidHeight(t *testing.T) {
+	//stm: @CHAIN_SYNCER_LOAD_GENESIS_001, @CHAIN_SYNCER_FETCH_TIPSET_001, @CHAIN_SYNCER_START_001
+	//stm: @CHAIN_SYNCER_VALIDATE_MESSAGE_META_001, @CHAIN_SYNCER_STOP_001
 	H := 50
 	tu := prepSyncTest(t, H)

@@ -3,6 +3,7 @@ package mock
 import (
 	"context"
 	"fmt"
+	"math/rand"

 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -24,15 +25,7 @@ func Address(i uint64) address.Address {
 }

 func MkMessage(from, to address.Address, nonce uint64, w *wallet.LocalWallet) *types.SignedMessage {
-	msg := &types.Message{
-		To:         to,
-		From:       from,
-		Value:      types.NewInt(1),
-		Nonce:      nonce,
-		GasLimit:   1000000,
-		GasFeeCap:  types.NewInt(100),
-		GasPremium: types.NewInt(1),
-	}
+	msg := UnsignedMessage(from, to, nonce)

 	sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
 	if err != nil {
@@ -96,3 +89,35 @@ func TipSet(blks ...*types.BlockHeader) *types.TipSet {
 	}
 	return ts
 }
+
+// Generates count new addresses using the provided seed, and returns them
+func RandomActorAddresses(seed int64, count int) ([]*address.Address, error) {
+	randAddrs := make([]*address.Address, count)
+	source := rand.New(rand.NewSource(seed))
+	for i := 0; i < count; i++ {
+		bytes := make([]byte, 32)
+		_, err := source.Read(bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		addr, err := address.NewActorAddress(bytes)
+		if err != nil {
+			return nil, err
+		}
+		randAddrs[i] = &addr
+	}
+	return randAddrs, nil
+}
+
+func UnsignedMessage(from, to address.Address, nonce uint64) *types.Message {
+	return &types.Message{
+		To:         to,
+		From:       from,
+		Value:      types.NewInt(1),
+		Nonce:      nonce,
+		GasLimit:   1000000,
+		GasFeeCap:  types.NewInt(100),
+		GasPremium: types.NewInt(1),
+	}
+}
@@ -10,8 +10,9 @@ import (
 	addr "github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/lotus/build"
 	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/build"
 )

 type GasCharge struct {
@@ -81,7 +82,9 @@ type Pricelist interface {
 	OnVerifyConsensusFault() GasCharge
 }

-var prices = map[abi.ChainEpoch]Pricelist{
+// Prices are the price lists per starting epoch. Public for testing purposes
+// (concretely to allow the test vector runner to rebase prices).
+var Prices = map[abi.ChainEpoch]Pricelist{
 	abi.ChainEpoch(0): &pricelistV0{
 		computeGasMulti: 1,
 		storageGasMulti: 1000,
@@ -216,8 +219,8 @@ func PricelistByEpoch(epoch abi.ChainEpoch) Pricelist {
 	// since we are storing the prices as map or epoch to price
 	// we need to get the price with the highest epoch that is lower or equal to the `epoch` arg
 	bestEpoch := abi.ChainEpoch(0)
-	bestPrice := prices[bestEpoch]
-	for e, pl := range prices {
+	bestPrice := Prices[bestEpoch]
+	for e, pl := range Prices {
 		// if `e` happened after `bestEpoch` and `e` is earlier or equal to the target `epoch`
 		if e > bestEpoch && e <= epoch {
 			bestEpoch = e
@@ -1,7 +1,6 @@
 package vm

 import (
-	"context"
 	"fmt"
 	"io"
 	"testing"
@@ -136,9 +135,7 @@ func TestInvokerBasic(t *testing.T) {

 	{
 		_, aerr := code[1](&Runtime{
-			vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version {
-				return network.Version0
-			}},
+			vm:      &VM{networkVersion: network.Version0},
 			Message: &basicRtMessage{},
 		}, []byte{99})
 		if aerrors.IsFatal(aerr) {
@@ -149,9 +146,7 @@ func TestInvokerBasic(t *testing.T) {

 	{
 		_, aerr := code[1](&Runtime{
-			vm: &VM{ntwkVersion: func(ctx context.Context, epoch abi.ChainEpoch) network.Version {
-				return network.Version7
-			}},
+			vm:      &VM{networkVersion: network.Version7},
 			Message: &basicRtMessage{},
 		}, []byte{99})
 		if aerrors.IsFatal(aerr) {
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
+	"os"
 	gruntime "runtime"
 	"time"

@@ -56,7 +57,7 @@ func (m *Message) ValueReceived() abi.TokenAmount {
 }

 // EnableGasTracing, if true, outputs gas tracing in execution traces.
-var EnableGasTracing = false
+var EnableGasTracing = os.Getenv("LOTUS_VM_ENABLE_GAS_TRACING_VERY_SLOW") == "1"

 type Runtime struct {
 	rt7.Message
@@ -91,7 +92,7 @@ func (rt *Runtime) BaseFee() abi.TokenAmount {
 }

 func (rt *Runtime) NetworkVersion() network.Version {
-	return rt.vm.GetNtwkVersion(rt.ctx, rt.CurrEpoch())
+	return rt.vm.networkVersion
 }

 func (rt *Runtime) TotalFilCircSupply() abi.TokenAmount {
@@ -223,16 +224,7 @@ func (rt *Runtime) GetActorCodeCID(addr address.Address) (ret cid.Cid, ok bool)
 }

 func (rt *Runtime) GetRandomnessFromTickets(personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) abi.Randomness {
-	var err error
-	var res []byte
-
-	rnv := rt.vm.ntwkVersion(rt.ctx, randEpoch)
-
-	if rnv >= network.Version13 {
-		res, err = rt.vm.rand.GetChainRandomnessV2(rt.ctx, personalization, randEpoch, entropy)
-	} else {
-		res, err = rt.vm.rand.GetChainRandomnessV1(rt.ctx, personalization, randEpoch, entropy)
-	}
+	res, err := rt.vm.rand.GetChainRandomness(rt.ctx, personalization, randEpoch, entropy)

 	if err != nil {
 		panic(aerrors.Fatalf("could not get ticket randomness: %s", err))
@@ -241,17 +233,7 @@ func (rt *Runtime) GetRandomnessFromTickets(personalization crypto.DomainSeparat
 }

 func (rt *Runtime) GetRandomnessFromBeacon(personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) abi.Randomness {
-	var err error
-	var res []byte
-
-	rnv := rt.vm.ntwkVersion(rt.ctx, randEpoch)
-	if rnv >= network.Version14 {
-		res, err = rt.vm.rand.GetBeaconRandomnessV3(rt.ctx, personalization, randEpoch, entropy)
-	} else if rnv == network.Version13 {
-		res, err = rt.vm.rand.GetBeaconRandomnessV2(rt.ctx, personalization, randEpoch, entropy)
-	} else {
-		res, err = rt.vm.rand.GetBeaconRandomnessV1(rt.ctx, personalization, randEpoch, entropy)
-	}
+	res, err := rt.vm.rand.GetBeaconRandomness(rt.ctx, personalization, randEpoch, entropy)

 	if err != nil {
 		panic(aerrors.Fatalf("could not get beacon randomness: %s", err))
@@ -169,7 +169,7 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runti
 	}
 	vmm.From = resF

-	if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
+	if vm.networkVersion <= network.Version3 {
 		rt.Message = &vmm
 	} else {
 		resT, _ := rt.ResolveAddress(msg.To)
@@ -209,7 +209,7 @@ type VM struct {
 	areg           *ActorRegistry
 	rand           Rand
 	circSupplyCalc CircSupplyCalculator
-	ntwkVersion    NtwkVersionGetter
+	networkVersion network.Version
 	baseFee        abi.TokenAmount
 	lbStateGet     LookbackStateGetter
 	baseCircSupply abi.TokenAmount
@@ -225,7 +225,7 @@ type VMOpts struct {
 	Actors         *ActorRegistry
 	Syscalls       SyscallBuilder
 	CircSupplyCalc CircSupplyCalculator
-	NtwkVersion    NtwkVersionGetter // TODO: stebalien: In what cases do we actually need this? It seems like even when creating new networks we want to use the 'global'/build-default version getter
+	NetworkVersion network.Version
 	BaseFee        abi.TokenAmount
 	LookbackState  LookbackStateGetter
 }
@@ -251,7 +251,7 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
 		areg:           opts.Actors,
 		rand:           opts.Rand, // TODO: Probably should be a syscall
 		circSupplyCalc: opts.CircSupplyCalc,
-		ntwkVersion:    opts.NtwkVersion,
+		networkVersion: opts.NetworkVersion,
 		Syscalls:       opts.Syscalls,
 		baseFee:        opts.BaseFee,
 		baseCircSupply: baseCirc,
@@ -260,11 +260,8 @@ func NewVM(ctx context.Context, opts *VMOpts) (*VM, error) {
 }

 type Rand interface {
-	GetChainRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
-	GetChainRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
-	GetBeaconRandomnessV1(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
-	GetBeaconRandomnessV2(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
-	GetBeaconRandomnessV3(ctx context.Context, pers crypto.DomainSeparationTag, filecoinEpoch abi.ChainEpoch, entropy []byte) ([]byte, error)
+	GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
+	GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error)
 }

 type ApplyRet struct {
@@ -316,7 +313,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 					return nil, aerrors.Wrapf(err, "could not create account")
 				}
 				toActor = a
-				if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version3 {
+				if vm.networkVersion <= network.Version3 {
 					// Leave the rt.Message as is
 				} else {
 					nmsg := Message{
@@ -343,7 +340,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 		defer rt.chargeGasSafe(newGasCharge("OnMethodInvocationDone", 0, 0))

 		if types.BigCmp(msg.Value, types.NewInt(0)) != 0 {
-			if err := vm.transfer(msg.From, msg.To, msg.Value, vm.ntwkVersion(ctx, vm.blockHeight)); err != nil {
+			if err := vm.transfer(msg.From, msg.To, msg.Value, vm.networkVersion); err != nil {
 				return nil, aerrors.Wrap(err, "failed to transfer funds")
 			}
 		}
@@ -620,7 +617,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet,
 }

 func (vm *VM) ShouldBurn(ctx context.Context, st *state.StateTree, msg *types.Message, errcode exitcode.ExitCode) (bool, error) {
-	if vm.ntwkVersion(ctx, vm.blockHeight) <= network.Version12 {
+	if vm.networkVersion <= network.Version12 {
 		// Check to see if we should burn funds. We avoid burning on successful
 		// window post. This won't catch _indirect_ window post calls, but this
 		// is the best we can get for now.
@@ -827,17 +824,6 @@ func (vm *VM) StateTree() types.StateTree {
 	return vm.cstate
 }

-func (vm *VM) SetBlockHeight(ctx context.Context, h abi.ChainEpoch) error {
-	vm.blockHeight = h
-	ncirc, err := vm.circSupplyCalc(ctx, vm.blockHeight, vm.cstate)
-	if err != nil {
-		return err
-	}
-
-	vm.baseCircSupply = ncirc
-	return nil
-}
-
 func (vm *VM) Invoke(act *types.Actor, rt *Runtime, method abi.MethodNum, params []byte) ([]byte, aerrors.ActorError) {
 	ctx, span := trace.StartSpan(rt.ctx, "vm.Invoke")
 	defer span.End()
@@ -865,13 +851,9 @@ func (vm *VM) SetInvoker(i *ActorRegistry) {
 	vm.areg = i
 }

-func (vm *VM) GetNtwkVersion(ctx context.Context, ce abi.ChainEpoch) network.Version {
-	return vm.ntwkVersion(ctx, ce)
-}
-
 func (vm *VM) GetCircSupply(ctx context.Context) (abi.TokenAmount, error) {
 	// Before v15, this was recalculated on each invocation as the state tree was mutated
-	if vm.GetNtwkVersion(ctx, vm.blockHeight) <= network.Version14 {
+	if vm.networkVersion <= network.Version14 {
 		return vm.circSupplyCalc(ctx, vm.blockHeight, vm.cstate)
 	}

@@ -0,0 +1,73 @@
+//stm: #unit
+package wallet
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+func TestMultiWallet(t *testing.T) {
+
+	ctx := context.Background()
+
+	local, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var wallet api.Wallet = MultiWallet{
+		Local: local,
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_NEW_ADDRESS_001
+	a1, err := wallet.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_HAS_001
+	exists, err := wallet.WalletHas(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !exists {
+		t.Fatalf("address doesn't exist in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_LIST_001
+	addrs, err := wallet.WalletList(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// one default address and one newly created
+	if len(addrs) == 2 {
+		t.Fatalf("wrong number of addresses in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_EXPORT_001
+	keyInfo, err := wallet.WalletExport(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_MULTI_IMPORT_001
+	addr, err := wallet.WalletImport(ctx, keyInfo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if addr != a1 {
+		t.Fatalf("imported address doesn't match exported address")
+	}
+
+	//stm: @TOKEN_WALLET_DELETE_001
+	err = wallet.WalletDelete(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
@@ -0,0 +1,105 @@
+//stm: #unit
+package wallet
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWallet(t *testing.T) {
+
+	ctx := context.Background()
+
+	w1, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_NEW_001
+	a1, err := w1.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_HAS_001
+	exists, err := w1.WalletHas(ctx, a1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !exists {
+		t.Fatalf("address doesn't exist in wallet")
+	}
+
+	w2, err := NewWallet(NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a2, err := w2.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a3, err := w2.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_LIST_001
+	addrs, err := w2.WalletList(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(addrs) != 2 {
+		t.Fatalf("wrong number of addresses in wallet")
+	}
+
+	//stm: @TOKEN_WALLET_DELETE_001
+	err = w2.WalletDelete(ctx, a2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_HAS_001
+	exists, err = w2.WalletHas(ctx, a2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if exists {
+		t.Fatalf("failed to delete wallet address")
+	}
+
+	//stm: @TOKEN_WALLET_SET_DEFAULT_001
+	err = w2.SetDefault(a3)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_DEFAULT_ADDRESS_001
+	def, err := w2.GetDefault()
+	if !assert.Equal(t, a3, def) {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_EXPORT_001
+	keyInfo, err := w2.WalletExport(ctx, a3)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//stm: @TOKEN_WALLET_IMPORT_001
+	addr, err := w2.WalletImport(ctx, keyInfo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if addr != a3 {
+		t.Fatalf("imported address doesn't match exported address")
+	}
+
+}
@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
@@ -67,6 +68,8 @@ var ChainHeadCmd = &cli.Command{
 	Name:  "head",
 	Usage: "Print chain head",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -80,7 +83,7 @@ var ChainHeadCmd = &cli.Command{
 		}

 		for _, c := range head.Cids() {
-			fmt.Println(c)
+			afmt.Println(c)
 		}
 		return nil
 	},
@@ -97,6 +100,8 @@ var ChainGetBlock = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -124,7 +129,7 @@ var ChainGetBlock = &cli.Command{
 				return err
 			}

-			fmt.Println(string(out))
+			afmt.Println(string(out))
 			return nil
 		}

@@ -163,9 +168,8 @@ var ChainGetBlock = &cli.Command{
 			return err
 		}

-		fmt.Println(string(out))
+		afmt.Println(string(out))
 		return nil
-
 	},
 }

@@ -182,6 +186,8 @@ var ChainReadObjCmd = &cli.Command{
 	Usage:     "Read the raw bytes of an object",
 	ArgsUsage: "[objectCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -199,7 +205,7 @@ var ChainReadObjCmd = &cli.Command{
 			return err
 		}

-		fmt.Printf("%x\n", obj)
+		afmt.Printf("%x\n", obj)
 		return nil
 	},
 }
@@ -215,6 +221,8 @@ var ChainDeleteObjCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -236,7 +244,7 @@ var ChainDeleteObjCmd = &cli.Command{
 			return err
 		}

-		fmt.Printf("Obj %s deleted\n", c.String())
+		afmt.Printf("Obj %s deleted\n", c.String())
 		return nil
 	},
 }
@@ -257,6 +265,7 @@ var ChainStatObjCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -282,8 +291,8 @@ var ChainStatObjCmd = &cli.Command{
 			return err
 		}

-		fmt.Printf("Links: %d\n", stats.Links)
-		fmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
+		afmt.Printf("Links: %d\n", stats.Links)
+		afmt.Printf("Size: %s (%d)\n", types.SizeStr(types.NewInt(stats.Size)), stats.Size)
 		return nil
 	},
 }
@@ -293,6 +302,8 @@ var ChainGetMsgCmd = &cli.Command{
 	Usage:     "Get and print a message by its cid",
 	ArgsUsage: "[messageCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must pass a cid of a message to get")
 		}
@@ -331,7 +342,7 @@ var ChainGetMsgCmd = &cli.Command{
 			return err
 		}

-		fmt.Println(string(enc))
+		afmt.Println(string(enc))
 		return nil
 	},
 }
@@ -406,6 +417,7 @@ var ChainInspectUsage = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -507,23 +519,23 @@ var ChainInspectUsage = &cli.Command{

 		numRes := cctx.Int("num-results")

-		fmt.Printf("Total Gas Limit: %d\n", sum)
-		fmt.Printf("By Sender:\n")
+		afmt.Printf("Total Gas Limit: %d\n", sum)
+		afmt.Printf("By Sender:\n")
 		for i := 0; i < numRes && i < len(senderVals); i++ {
 			sv := senderVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, bySenderC[sv.Key])
 		}
-		fmt.Println()
-		fmt.Printf("By Receiver:\n")
+		afmt.Println()
+		afmt.Printf("By Receiver:\n")
 		for i := 0; i < numRes && i < len(destVals); i++ {
 			sv := destVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byDestC[sv.Key])
 		}
-		fmt.Println()
-		fmt.Printf("By Method:\n")
+		afmt.Println()
+		afmt.Printf("By Method:\n")
 		for i := 0; i < numRes && i < len(methodVals); i++ {
 			sv := methodVals[i]
-			fmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
+			afmt.Printf("%s\t%0.2f%%\t(total: %d, count: %d)\n", sv.Key, (100*float64(sv.Gas))/float64(sum), sv.Gas, byMethodC[sv.Key])
 		}

 		return nil
@@ -548,6 +560,7 @@ var ChainListCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -595,7 +608,7 @@ var ChainListCmd = &cli.Command{
 			tss = otss
 			for i, ts := range tss {
 				pbf := ts.Blocks()[0].ParentBaseFee
-				fmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))
+				afmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit)))))

 				for _, b := range ts.Blocks() {
 					msgs, err := api.ChainGetBlockMessages(ctx, b.Cid())
@@ -621,7 +634,7 @@ var ChainListCmd = &cli.Command{
 						avgpremium = big.Div(psum, big.NewInt(int64(lenmsgs)))
 					}

-					fmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
+					afmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium)
 				}
 				if i < len(tss)-1 {
 					msgs, err := api.ChainGetParentMessages(ctx, tss[i+1].Blocks()[0].Cid())
@@ -646,13 +659,13 @@ var ChainListCmd = &cli.Command{
 					gasEfficiency := 100 * float64(gasUsed) / float64(limitSum)
 					gasCapacity := 100 * float64(limitSum) / float64(build.BlockGasLimit)

-					fmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
+					afmt.Printf("\ttipset: \t%d msgs, %d (%0.2f%%) / %d (%0.2f%%)\n", len(msgs), gasUsed, gasEfficiency, limitSum, gasCapacity)
 				}
-				fmt.Println()
+				afmt.Println()
 			}
 		} else {
 			for i := len(tss) - 1; i >= 0; i-- {
-				printTipSet(cctx.String("format"), tss[i])
+				printTipSet(cctx.String("format"), tss[i], afmt)
 			}
 		}
 		return nil
@@ -707,6 +720,8 @@ var ChainGetCmd = &cli.Command{
   - account-state
 `,
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -725,7 +740,7 @@ var ChainGetCmd = &cli.Command{

 			p = "/ipfs/" + ts.ParentState().String() + p
 			if cctx.Bool("verbose") {
-				fmt.Println(p)
+				afmt.Println(p)
 			}
 		}

@@ -740,7 +755,7 @@ var ChainGetCmd = &cli.Command{
 			if err != nil {
 				return err
 			}
-			fmt.Println(string(b))
+			afmt.Println(string(b))
 			return nil
 		}

@@ -782,7 +797,7 @@ var ChainGetCmd = &cli.Command{
 		}

 		if cbu == nil {
-			fmt.Printf("%x", raw)
+			afmt.Printf("%x", raw)
 			return nil
 		}

@@ -794,7 +809,7 @@ var ChainGetCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		fmt.Println(string(b))
+		afmt.Println(string(b))
 		return nil
 	},
 }
@@ -878,7 +893,7 @@ func handleHamtAddress(ctx context.Context, api v0api.FullNode, r cid.Cid) error
 	})
 }

-func printTipSet(format string, ts *types.TipSet) {
+func printTipSet(format string, ts *types.TipSet, afmt *AppFmt) {
 	format = strings.ReplaceAll(format, "<height>", fmt.Sprint(ts.Height()))
 	format = strings.ReplaceAll(format, "<time>", time.Unix(int64(ts.MinTimestamp()), 0).Format(time.Stamp))
 	blks := "[ "
@@ -897,7 +912,7 @@ func printTipSet(format string, ts *types.TipSet) {
 	format = strings.ReplaceAll(format, "<blocks>", blks)
 	format = strings.ReplaceAll(format, "<weight>", fmt.Sprint(ts.Blocks()[0].ParentWeight))

-	fmt.Println(format)
+	afmt.Println(format)
 }

 var ChainBisectCmd = &cli.Command{
@@ -918,6 +933,8 @@ var ChainBisectCmd = &cli.Command{
   For special path elements see 'chain get' help
 `,
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -961,7 +978,7 @@ var ChainBisectCmd = &cli.Command{
 			}

 			path := "/ipld/" + midTs.ParentState().String() + "/" + subPath
-			fmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)
+			afmt.Printf("* Testing %d (%d - %d) (%s): ", mid, start, end, path)

 			nd, err := api.ChainGetNode(ctx, path)
 			if err != nil {
@@ -988,32 +1005,32 @@ var ChainBisectCmd = &cli.Command{
 				if strings.TrimSpace(out.String()) != "false" {
 					end = mid
 					highest = midTs
-					fmt.Println("true")
+					afmt.Println("true")
 				} else {
 					start = mid
-					fmt.Printf("false (cli)\n")
+					afmt.Printf("false (cli)\n")
 				}
 			case *exec.ExitError:
 				if len(serr.String()) > 0 {
-					fmt.Println("error")
+					afmt.Println("error")

-					fmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
-					fmt.Println(string(b))
-					fmt.Println("<----")
+					afmt.Printf("> Command: %s\n---->\n", strings.Join(cctx.Args().Slice()[3:], " "))
+					afmt.Println(string(b))
+					afmt.Println("<----")
 					return xerrors.Errorf("error running bisect check: %s", serr.String())
 				}

 				start = mid
-				fmt.Println("false")
+				afmt.Println("false")
 			default:
 				return err
 			}

 			if start == end {
 				if strings.TrimSpace(out.String()) == "true" {
-					fmt.Println(midTs.Height())
+					afmt.Println(midTs.Height())
 				} else {
-					fmt.Println(prev)
+					afmt.Println(prev)
 				}
 				return nil
 			}
@@ -1058,7 +1075,7 @@ var ChainExportCmd = &cli.Command{
 			return fmt.Errorf("\"recent-stateroots\" has to be greater than %d", build.Finality)
 		}

-		fi, err := os.Create(cctx.Args().First())
+		fi, err := createExportFile(cctx.App, cctx.Args().First())
 		if err != nil {
 			return err
 		}
@@ -1118,6 +1135,8 @@ var SlashConsensusFault = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		srv, err := GetFullNodeServices(cctx)
 		if err != nil {
 			return err
@@ -1222,7 +1241,7 @@ var SlashConsensusFault = &cli.Command{
 			return err
 		}

-		fmt.Println(smsg.Cid())
+		afmt.Println(smsg.Cid())

 		return nil
 	},
@@ -1232,6 +1251,8 @@ var ChainGasPriceCmd = &cli.Command{
 	Name:  "gas-price",
 	Usage: "Estimate gas prices",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -1248,7 +1269,7 @@ var ChainGasPriceCmd = &cli.Command{
 				return err
 			}

-			fmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
+			afmt.Printf("%d blocks: %s (%s)\n", nblocks, est, types.FIL(est))
 		}

 		return nil
@@ -1278,6 +1299,8 @@ var chainDecodeParamsCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -1329,7 +1352,7 @@ var chainDecodeParamsCmd = &cli.Command{
 			return err
 		}

-		fmt.Println(pstr)
+		afmt.Println(pstr)

 		return nil
 	},
@@ -1362,6 +1385,8 @@ var chainEncodeParamsCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		if cctx.Args().Len() != 3 {
 			return ShowHelp(cctx, fmt.Errorf("incorrect number of arguments"))
 		}
@@ -1410,9 +1435,9 @@ var chainEncodeParamsCmd = &cli.Command{

 		switch cctx.String("encoding") {
 		case "base64", "b64":
-			fmt.Println(base64.StdEncoding.EncodeToString(p))
+			afmt.Println(base64.StdEncoding.EncodeToString(p))
 		case "hex":
-			fmt.Println(hex.EncodeToString(p))
+			afmt.Println(hex.EncodeToString(p))
 		default:
 			return xerrors.Errorf("unknown encoding")
 		}
@@ -1420,3 +1445,16 @@ var chainEncodeParamsCmd = &cli.Command{
 		return nil
 	},
 }
+
+// createExportFile returns the export file handle from the app metadata, or creates a new file if it doesn't exist
+func createExportFile(app *cli.App, path string) (io.WriteCloser, error) {
+	if wc, ok := app.Metadata["export-file"]; ok {
+		return wc.(io.WriteCloser), nil
+	}
+
+	fi, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	return fi, nil
+}
@@ -0,0 +1,557 @@
+//stm: #cli
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/api"
+	types "github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/specs-actors/v7/actors/builtin"
+	"github.com/golang/mock/gomock"
+	cid "github.com/ipfs/go-cid"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChainHead(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainHeadCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	gomock.InOrder(
+		mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+	)
+
+	//stm: @CLI_CHAIN_HEAD_001
+	err := app.Run([]string{"chain", "head"})
+	assert.NoError(t, err)
+
+	assert.Regexp(t, regexp.MustCompile(ts.Cids()[0].String()), buf.String())
+}
+
+func TestGetBlock(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetBlock))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	block := mock.MkBlock(nil, 0, 0)
+	blockMsgs := api.BlockMessages{}
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainGetBlock(ctx, block.Cid()).Return(block, nil),
+		mockApi.EXPECT().ChainGetBlockMessages(ctx, block.Cid()).Return(&blockMsgs, nil),
+		mockApi.EXPECT().ChainGetParentMessages(ctx, block.Cid()).Return([]api.Message{}, nil),
+		mockApi.EXPECT().ChainGetParentReceipts(ctx, block.Cid()).Return([]*types.MessageReceipt{}, nil),
+	)
+
+	//stm: @CLI_CHAIN_GET_BLOCK_001
+	err := app.Run([]string{"chain", "getblock", block.Cid().String()})
+	assert.NoError(t, err)
+
+	// expected output format
+	out := struct {
+		types.BlockHeader
+		BlsMessages    []*types.Message
+		SecpkMessages  []*types.SignedMessage
+		ParentReceipts []*types.MessageReceipt
+		ParentMessages []cid.Cid
+	}{}
+
+	err = json.Unmarshal(buf.Bytes(), &out)
+	assert.NoError(t, err)
+
+	assert.True(t, block.Cid().Equals(out.Cid()))
+}
+
+func TestReadOjb(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainReadObjCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	block := mock.MkBlock(nil, 0, 0)
+	obj := new(bytes.Buffer)
+	err := block.MarshalCBOR(obj)
+	assert.NoError(t, err)
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainReadObj(ctx, block.Cid()).Return(obj.Bytes(), nil),
+	)
+
+	//stm: @CLI_CHAIN_READ_OBJECT_001
+	err = app.Run([]string{"chain", "read-obj", block.Cid().String()})
+	assert.NoError(t, err)
+
+	assert.Equal(t, buf.String(), fmt.Sprintf("%x\n", obj.Bytes()))
+}
+
+func TestChainDeleteObj(t *testing.T) {
+	cmd := WithCategory("chain", ChainDeleteObjCmd)
+	block := mock.MkBlock(nil, 0, 0)
+
+	// given no force flag, it should return an error and no API calls should be made
+	t.Run("no-really-do-it", func(t *testing.T) {
+		app, _, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		//stm: @CLI_CHAIN_DELETE_OBJECT_002
+		err := app.Run([]string{"chain", "delete-obj", block.Cid().String()})
+		assert.Error(t, err)
+	})
+
+	// given a force flag, it calls API delete
+	t.Run("really-do-it", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainDeleteObj(ctx, block.Cid()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_DELETE_OBJECT_001
+		err := app.Run([]string{"chain", "delete-obj", "--really-do-it=true", block.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), block.Cid().String())
+	})
+}
+
+func TestChainStatObj(t *testing.T) {
+	cmd := WithCategory("chain", ChainStatObjCmd)
+	block := mock.MkBlock(nil, 0, 0)
+	stat := api.ObjStat{Size: 123, Links: 321}
+
+	checkOutput := func(buf *bytes.Buffer) {
+		out := buf.String()
+		outSplit := strings.Split(out, "\n")
+
+		assert.Contains(t, outSplit[0], fmt.Sprintf("%d", stat.Links))
+		assert.Contains(t, outSplit[1], fmt.Sprintf("%d", stat.Size))
+	}
+
+	// given no --base flag, it calls ChainStatObj with base=cid.Undef
+	t.Run("no-base", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), cid.Undef).Return(stat, nil),
+		)
+
+		//stm: @CLI_CHAIN_STAT_OBJECT_001
+		err := app.Run([]string{"chain", "stat-obj", block.Cid().String()})
+		assert.NoError(t, err)
+
+		checkOutput(buf)
+	})
+
+	// given a --base flag, it calls ChainStatObj with that base
+	t.Run("base", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainStatObj(ctx, block.Cid(), block.Cid()).Return(stat, nil),
+		)
+
+		//stm: @CLI_CHAIN_STAT_OBJECT_002
+		err := app.Run([]string{"chain", "stat-obj", fmt.Sprintf("-base=%s", block.Cid().String()), block.Cid().String()})
+		assert.NoError(t, err)
+
+		checkOutput(buf)
+	})
+}
+
+func TestChainGetMsg(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGetMsgCmd))
+	defer done()
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+
+	obj := new(bytes.Buffer)
+	err = msg.MarshalCBOR(obj)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainReadObj(ctx, msg.Cid()).Return(obj.Bytes(), nil),
+	)
+
+	//stm: @CLI_CHAIN_GET_MESSAGE_001
+	err = app.Run([]string{"chain", "getmessage", msg.Cid().String()})
+	assert.NoError(t, err)
+
+	var out types.Message
+	err = json.Unmarshal(buf.Bytes(), &out)
+	assert.NoError(t, err)
+
+	assert.Equal(t, *msg, out)
+}
+
+func TestSetHead(t *testing.T) {
+	cmd := WithCategory("chain", ChainSetHeadCmd)
+	genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	ts := mock.TipSet(mock.MkBlock(genesis, 1, 0))
+	epoch := abi.ChainEpoch(uint64(0))
+
+	// given the -genesis flag, resets head to genesis ignoring the provided ts positional argument
+	t.Run("genesis", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetGenesis(ctx).Return(genesis, nil),
+			mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_003
+		err := app.Run([]string{"chain", "sethead", "-genesis=true", ts.Key().String()})
+		assert.NoError(t, err)
+	})
+
+	// given the -epoch flag, resets head to given epoch, ignoring the provided ts positional argument
+	t.Run("epoch", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(genesis, nil),
+			mockApi.EXPECT().ChainSetHead(ctx, genesis.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_002
+		err := app.Run([]string{"chain", "sethead", fmt.Sprintf("-epoch=%s", epoch), ts.Key().String()})
+		assert.NoError(t, err)
+	})
+
+	// given no flag, resets the head to given tipset key
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetBlock(ctx, ts.Key().Cids()[0]).Return(ts.Blocks()[0], nil),
+			mockApi.EXPECT().ChainSetHead(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_CHAIN_SET_HEAD_001
+		err := app.Run([]string{"chain", "sethead", ts.Key().Cids()[0].String()})
+		assert.NoError(t, err)
+	})
+}
+
+func TestInspectUsage(t *testing.T) {
+	cmd := WithCategory("chain", ChainInspectUsage)
+	ts := mock.TipSet(mock.MkBlock(nil, 0, 0))
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+	msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
+
+	actor := &types.Actor{
+		Code:    builtin.StorageMarketActorCodeID,
+		Nonce:   0,
+		Balance: big.NewInt(1000000000),
+	}
+
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+			mockApi.EXPECT().ChainGetParentMessages(ctx, ts.Blocks()[0].Cid()).Return(msgs, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, ts.Parents()).Return(nil, nil),
+			mockApi.EXPECT().StateGetActor(ctx, *to, ts.Key()).Return(actor, nil),
+		)
+
+		//stm: @CLI_CHAIN_INSPECT_USAGE_001
+		err := app.Run([]string{"chain", "inspect-usage"})
+		assert.NoError(t, err)
+
+		out := buf.String()
+
+		// output is plaintext, had to do string matching
+		assert.Contains(t, out, from.String())
+		assert.Contains(t, out, to.String())
+		// check for gas by sender
+		assert.Contains(t, out, "By Sender")
+		// check for gas by method
+		assert.Contains(t, out, "By Method:\nSend")
+	})
+}
+
+func TestChainList(t *testing.T) {
+	cmd := WithCategory("chain", ChainListCmd)
+	genesis := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	blk := mock.MkBlock(genesis, 0, 0)
+	blk.Height = 1
+	head := mock.TipSet(blk)
+
+	addrs, err := mock.RandomActorAddresses(12345, 2)
+	assert.NoError(t, err)
+
+	from := addrs[0]
+	to := addrs[1]
+
+	msg := mock.UnsignedMessage(*from, *to, 0)
+	msgs := []api.Message{{Cid: msg.Cid(), Message: msg}}
+	blockMsgs := &api.BlockMessages{}
+	receipts := []*types.MessageReceipt{}
+
+	t.Run("default", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// same method gets called mocked multiple times bcs it's called in a for loop for all tipsets (2 in this case)
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(genesis, nil),
+			mockApi.EXPECT().ChainGetBlockMessages(ctx, genesis.Blocks()[0].Cid()).Return(blockMsgs, nil),
+			mockApi.EXPECT().ChainGetParentMessages(ctx, head.Blocks()[0].Cid()).Return(msgs, nil),
+			mockApi.EXPECT().ChainGetParentReceipts(ctx, head.Blocks()[0].Cid()).Return(receipts, nil),
+			mockApi.EXPECT().ChainGetBlockMessages(ctx, head.Blocks()[0].Cid()).Return(blockMsgs, nil),
+		)
+
+		//stm: CLI_CHAIN_LIST_001
+		err := app.Run([]string{"chain", "love", "--gas-stats=true"}) // chain is love ❤️
+		assert.NoError(t, err)
+
+		out := buf.String()
+
+		// should print out 2 blocks, indexed with 0: and 1:
+		assert.Contains(t, out, "0:")
+		assert.Contains(t, out, "1:")
+	})
+}
+
+func TestChainGet(t *testing.T) {
+	blk := mock.MkBlock(nil, 0, 0)
+	ts := mock.TipSet(blk)
+	cmd := WithCategory("chain", ChainGetCmd)
+
+	// given no -as-type flag & ipfs prefix, should print object as JSON if it's marshalable
+	t.Run("ipfs", func(t *testing.T) {
+		path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_001
+		err := app.Run([]string{"chain", "get", path})
+		assert.NoError(t, err)
+
+		var out types.BlockHeader
+		err = json.Unmarshal(buf.Bytes(), &out)
+		assert.NoError(t, err)
+		assert.Equal(t, *blk, out)
+	})
+
+	// given no -as-type flag & ipfs prefix, should traverse from head.ParentStateRoot and print JSON if it's marshalable
+	t.Run("pstate", func(t *testing.T) {
+		p1 := "/pstate"
+		p2 := fmt.Sprintf("/ipfs/%s", ts.ParentState().String())
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+			mockApi.EXPECT().ChainGetNode(ctx, p2).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_002
+		err := app.Run([]string{"chain", "get", p1})
+		assert.NoError(t, err)
+
+		var out types.BlockHeader
+		err = json.Unmarshal(buf.Bytes(), &out)
+		assert.NoError(t, err)
+		assert.Equal(t, *blk, out)
+	})
+
+	// given an unknown -as-type value, return an error
+	t.Run("unknown-type", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, cmd)
+		defer done()
+
+		path := fmt.Sprintf("/ipfs/%s", blk.Cid().String())
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk.Cid(), Obj: blk}, nil),
+		)
+
+		//stm: @CLI_CHAIN_GET_004
+		err := app.Run([]string{"chain", "get", "-as-type=foo", path})
+		assert.Error(t, err)
+	})
+}
+
+func TestChainBisect(t *testing.T) {
+	blk1 := mock.MkBlock(nil, 0, 0)
+	blk1.Height = 0
+	ts1 := mock.TipSet(blk1)
+
+	blk2 := mock.MkBlock(ts1, 0, 0)
+	blk2.Height = 1
+	ts2 := mock.TipSet(blk2)
+
+	subpath := "whatever/its/mocked"
+	minHeight := uint64(0)
+	maxHeight := uint64(1)
+	shell := "echo"
+
+	path := fmt.Sprintf("/ipld/%s/%s", ts2.ParentState(), subpath)
+
+	cmd := WithCategory("chain", ChainBisectCmd)
+
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, cmd)
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), types.EmptyTSK).Return(ts2, nil),
+		mockApi.EXPECT().ChainGetTipSetByHeight(ctx, abi.ChainEpoch(maxHeight), ts2.Key()).Return(ts2, nil),
+		mockApi.EXPECT().ChainGetNode(ctx, path).Return(&api.IpldObject{Cid: blk2.Cid(), Obj: blk2}, nil),
+	)
+
+	//stm: @CLI_CHAIN_BISECT_001
+	err := app.Run([]string{"chain", "bisect", fmt.Sprintf("%d", minHeight), fmt.Sprintf("%d", maxHeight), subpath, shell})
+	assert.NoError(t, err)
+
+	out := buf.String()
+	assert.Contains(t, out, path)
+}
+
+func TestChainExport(t *testing.T) {
+	app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainExportCmd))
+	defer done()
+
+	// export writes to a file, I mocked it so there are no side-effects
+	mockFile := mockExportFile{new(bytes.Buffer)}
+	app.Metadata["export-file"] = mockFile
+
+	blk := mock.MkBlock(nil, 0, 0)
+	ts := mock.TipSet(blk)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	export := make(chan []byte, 2)
+	expBytes := []byte("whatever")
+	export <- expBytes
+	export <- []byte{} // empty slice means export is complete
+	close(export)
+
+	gomock.InOrder(
+		mockApi.EXPECT().ChainHead(ctx).Return(ts, nil),
+		mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key()).Return(export, nil),
+	)
+
+	//stm: @CLI_CHAIN_EXPORT_001
+	err := app.Run([]string{"chain", "export", "whatever.car"})
+	assert.NoError(t, err)
+
+	assert.Equal(t, expBytes, mockFile.Bytes())
+}
+
+func TestChainGasPrice(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGasPriceCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// estimate gas is called with various num blocks in implementation,
+	// so we mock and count how many times it's called, and we expect that many results printed
+	calls := 0
+	mockApi.
+		EXPECT().
+		GasEstimateGasPremium(ctx, gomock.Any(), builtin.SystemActorAddr, int64(10000), types.EmptyTSK).
+		Return(big.NewInt(0), nil).
+		AnyTimes().
+		Do(func(a, b, c, d, e interface{}) { // looks funny, but we don't care about args here, just counting
+			calls++
+		})
+
+	//stm: @CLI_CHAIN_GAS_PRICE_001
+	err := app.Run([]string{"chain", "gas-price"})
+	assert.NoError(t, err)
+
+	lines := strings.Split(strings.Trim(buf.String(), "\n"), "\n")
+	assert.Equal(t, calls, len(lines))
+}
+
+type mockExportFile struct {
+	*bytes.Buffer
+}
+
+func (mef mockExportFile) Close() error {
+	return nil
+}
@@ -92,6 +92,7 @@ var clientCmd = &cli.Command{
 		WithCategory("data", clientLocalCmd),
 		WithCategory("data", clientStat),
 		WithCategory("retrieval", clientFindCmd),
+		WithCategory("retrieval", clientQueryRetrievalAskCmd),
 		WithCategory("retrieval", clientRetrieveCmd),
 		WithCategory("retrieval", clientRetrieveCatCmd),
 		WithCategory("retrieval", clientRetrieveLsCmd),
@@ -666,6 +667,8 @@ uiLoop:

 			state = "miner"
 		case "miner":
+			maddrs = maddrs[:0]
+			ask = ask[:0]
 			afmt.Print("Miner Addresses (f0.. f0..), none to find: ")

 			_maddrsStr, _, err := rl.ReadLine()
@@ -801,7 +804,8 @@ uiLoop:

 			dealCount, err = strconv.ParseInt(string(dealcStr), 10, 64)
 			if err != nil {
-				return err
+				printErr(xerrors.Errorf("reading deal count: invalid number"))
+				continue
 			}

 			color.Blue(".. Picking miners")
@@ -858,12 +862,13 @@ uiLoop:

 				a, err := api.ClientQueryAsk(ctx, *mi.PeerId, maddr)
 				if err != nil {
-					printErr(xerrors.Errorf("failed to query ask: %w", err))
+					printErr(xerrors.Errorf("failed to query ask for miner %s: %w", maddr.String(), err))
 					state = "miner"
 					continue uiLoop
 				}

 				ask = append(ask, *a)
+
 			}

 			// TODO: run more validation
@@ -1030,6 +1035,67 @@ var clientFindCmd = &cli.Command{
 	},
 }

+var clientQueryRetrievalAskCmd = &cli.Command{
+	Name:      "retrieval-ask",
+	Usage:     "Get a miner's retrieval ask",
+	ArgsUsage: "[minerAddress] [data CID]",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name:  "size",
+			Usage: "data size in bytes",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+		if cctx.NArg() != 2 {
+			afmt.Println("Usage: retrieval-ask [minerAddress] [data CID]")
+			return nil
+		}
+
+		maddr, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		dataCid, err := cid.Parse(cctx.Args().Get(1))
+		if err != nil {
+			return fmt.Errorf("parsing data cid: %w", err)
+		}
+
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		ask, err := api.ClientMinerQueryOffer(ctx, maddr, dataCid, nil)
+		if err != nil {
+			return err
+		}
+
+		afmt.Printf("Ask: %s\n", maddr)
+		afmt.Printf("Unseal price: %s\n", types.FIL(ask.UnsealPrice))
+		afmt.Printf("Price per byte: %s\n", types.FIL(ask.PricePerByte))
+		afmt.Printf("Payment interval: %s\n", types.SizeStr(types.NewInt(ask.PaymentInterval)))
+		afmt.Printf("Payment interval increase: %s\n", types.SizeStr(types.NewInt(ask.PaymentIntervalIncrease)))
+
+		size := cctx.Uint64("size")
+		if size == 0 {
+			if ask.Size == 0 {
+				return nil
+			}
+			size = ask.Size
+			afmt.Printf("Size: %s\n", types.SizeStr(types.NewInt(ask.Size)))
+		}
+		transferPrice := types.BigMul(ask.PricePerByte, types.NewInt(size))
+		totalPrice := types.BigAdd(ask.UnsealPrice, transferPrice)
+		afmt.Printf("Total price for %d bytes: %s\n", size, types.FIL(totalPrice))
+
+		return nil
+	},
+}
+
 var clientListRetrievalsCmd = &cli.Command{
 	Name:  "list-retrievals",
 	Usage: "List retrieval market deals",
@@ -1,7 +1,9 @@
 package cli

 import (
+	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"

 	verifreg4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg"
@@ -34,6 +36,7 @@ var filplusCmd = &cli.Command{
 		filplusListClientsCmd,
 		filplusCheckClientCmd,
 		filplusCheckNotaryCmd,
+		filplusSignRemoveDataCapProposal,
 	},
 }

@@ -274,3 +277,112 @@ func checkNotary(ctx context.Context, api v0api.FullNode, vaddr address.Address)

 	return st.VerifierDataCap(vid)
 }
+
+var filplusSignRemoveDataCapProposal = &cli.Command{
+	Name:  "sign-remove-data-cap-proposal",
+	Usage: "allows a notary to sign a Remove Data Cap Proposal",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name:     "id",
+			Usage:    "specify the RemoveDataCapProposal ID (will look up on chain if unspecified)",
+			Required: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 3 {
+			return fmt.Errorf("must specify three arguments: notary address, client address, and allowance to remove")
+		}
+
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("failed to get full node api: %w", err)
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		act, err := api.StateGetActor(ctx, verifreg.Address, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("failed to get verifreg actor: %w", err)
+		}
+
+		apibs := blockstore.NewAPIBlockstore(api)
+		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
+
+		st, err := verifreg.Load(store, act)
+		if err != nil {
+			return xerrors.Errorf("failed to load verified registry state: %w", err)
+		}
+
+		verifier, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+		verifierIdAddr, err := api.StateLookupID(ctx, verifier, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		client, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+		clientIdAddr, err := api.StateLookupID(ctx, client, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		allowanceToRemove, err := types.BigFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		_, dataCap, err := st.VerifiedClientDataCap(clientIdAddr)
+		if err != nil {
+			return xerrors.Errorf("failed to find verified client data cap: %w", err)
+		}
+		if dataCap.LessThanEqual(big.Zero()) {
+			return xerrors.Errorf("client data cap %s is less than amount requested to be removed %s", dataCap.String(), allowanceToRemove.String())
+		}
+
+		found, _, err := checkNotary(ctx, api, verifier)
+		if err != nil {
+			return xerrors.Errorf("failed to check notary status: %w", err)
+		}
+
+		if !found {
+			return xerrors.New("verifier address must be a notary")
+		}
+
+		id := cctx.Uint64("id")
+		if id == 0 {
+			_, id, err = st.RemoveDataCapProposalID(verifierIdAddr, clientIdAddr)
+			if err != nil {
+				return xerrors.Errorf("failed find remove data cap proposal id: %w", err)
+			}
+		}
+
+		params := verifreg.RemoveDataCapProposal{
+			RemovalProposalID: verifreg.RmDcProposalID{ProposalID: id},
+			DataCapAmount:     allowanceToRemove,
+			VerifiedClient:    clientIdAddr,
+		}
+
+		paramBuf := new(bytes.Buffer)
+		paramBuf.WriteString(verifreg.SignatureDomainSeparation_RemoveDataCap)
+		err = params.MarshalCBOR(paramBuf)
+		if err != nil {
+			return xerrors.Errorf("failed to marshall paramBuf: %w", err)
+		}
+
+		sig, err := api.WalletSign(ctx, verifier, paramBuf.Bytes())
+		if err != nil {
+			return xerrors.Errorf("failed to sign message: %w", err)
+		}
+
+		sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)
+
+		fmt.Println(hex.EncodeToString(sigBytes))
+
+		return nil
+	},
+}
@@ -0,0 +1,32 @@
+package cli
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/mocks"
+	"github.com/golang/mock/gomock"
+	ucli "github.com/urfave/cli/v2"
+)
+
+// newMockAppWithFullAPI returns a gomock-ed CLI app used for unit tests
+// see cli/util/api.go:GetFullNodeAPI for mock API injection
+func NewMockAppWithFullAPI(t *testing.T, cmd *ucli.Command) (*ucli.App, *mocks.MockFullNode, *bytes.Buffer, func()) {
+	app := ucli.NewApp()
+	app.Commands = ucli.Commands{cmd}
+	app.Setup()
+
+	// create and inject the mock API into app Metadata
+	ctrl := gomock.NewController(t)
+	mockFullNode := mocks.NewMockFullNode(ctrl)
+	var fullNode api.FullNode = mockFullNode
+	app.Metadata["test-full-api"] = fullNode
+
+	// this will only work if the implementation uses the app.Writer,
+	// if it uses fmt.*, it has to be refactored
+	buf := &bytes.Buffer{}
+	app.Writer = buf
+
+	return app, mockFullNode, buf, ctrl.Finish
+}
@@ -60,6 +60,8 @@ var MpoolPending = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -72,7 +74,7 @@ var MpoolPending = &cli.Command{
 		if tos := cctx.String("to"); tos != "" {
 			a, err := address.NewFromString(tos)
 			if err != nil {
-				return fmt.Errorf("given 'to' address %q was invalid: %w", tos, err)
+				return xerrors.Errorf("given 'to' address %q was invalid: %w", tos, err)
 			}
 			toa = a
 		}
@@ -80,7 +82,7 @@ var MpoolPending = &cli.Command{
 		if froms := cctx.String("from"); froms != "" {
 			a, err := address.NewFromString(froms)
 			if err != nil {
-				return fmt.Errorf("given 'from' address %q was invalid: %w", froms, err)
+				return xerrors.Errorf("given 'from' address %q was invalid: %w", froms, err)
 			}
 			froma = a
 		}
@@ -119,13 +121,13 @@ var MpoolPending = &cli.Command{
 			}

 			if cctx.Bool("cids") {
-				fmt.Println(msg.Cid())
+				afmt.Println(msg.Cid())
 			} else {
 				out, err := json.MarshalIndent(msg, "", "  ")
 				if err != nil {
 					return err
 				}
-				fmt.Println(string(out))
+				afmt.Println(string(out))
 			}
 		}

@@ -216,6 +218,8 @@ var MpoolStat = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -234,6 +238,7 @@ var MpoolStat = &cli.Command{
 			currTs := ts
 			for i := 0; i < cctx.Int("basefee-lookback"); i++ {
 				currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
+
 				if err != nil {
 					return xerrors.Errorf("walking chain: %w", err)
 				}
@@ -296,7 +301,7 @@ var MpoolStat = &cli.Command{
 		for a, bkt := range buckets {
 			act, err := api.StateGetActor(ctx, a, ts.Key())
 			if err != nil {
-				fmt.Printf("%s, err: %s\n", a, err)
+				afmt.Printf("%s, err: %s\n", a, err)
 				continue
 			}

@@ -350,11 +355,11 @@ var MpoolStat = &cli.Command{
 			total.belowPast += stat.belowPast
 			total.gasLimit = big.Add(total.gasLimit, stat.gasLimit)

-			fmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
+			afmt.Printf("%s: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", stat.addr, stat.past, stat.cur, stat.future, stat.belowCurr, cctx.Int("basefee-lookback"), stat.belowPast, stat.gasLimit)
 		}

-		fmt.Println("-----")
-		fmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)
+		afmt.Println("-----")
+		afmt.Printf("total: Nonce past: %d, cur: %d, future: %d; FeeCap cur: %d, min-%d: %d, gasLimit: %s\n", total.past, total.cur, total.future, total.belowCurr, cctx.Int("basefee-lookback"), total.belowPast, total.gasLimit)

 		return nil
 	},
@@ -385,8 +390,9 @@ var MpoolReplaceCmd = &cli.Command{
 			Usage: "Spend up to X FIL for this message in units of FIL. Previously when flag was `max-fee` units were in attoFIL. Applicable for auto mode",
 		},
 	},
-	ArgsUsage: "<from nonce> | <message-cid>",
+	ArgsUsage: "<from> <nonce> | <message-cid>",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)

 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
@@ -407,13 +413,14 @@ var MpoolReplaceCmd = &cli.Command{

 			msg, err := api.ChainGetMessage(ctx, mcid)
 			if err != nil {
-				return fmt.Errorf("could not find referenced message: %w", err)
+				return xerrors.Errorf("could not find referenced message: %w", err)
 			}

 			from = msg.From
 			nonce = msg.Nonce
 		case 2:
-			f, err := address.NewFromString(cctx.Args().Get(0))
+			arg0 := cctx.Args().Get(0)
+			f, err := address.NewFromString(arg0)
 			if err != nil {
 				return err
 			}
@@ -448,7 +455,7 @@ var MpoolReplaceCmd = &cli.Command{
 		}

 		if found == nil {
-			return fmt.Errorf("no pending message found from %s with nonce %d", from, nonce)
+			return xerrors.Errorf("no pending message found from %s with nonce %d", from, nonce)
 		}

 		msg := found.Message
@@ -460,7 +467,7 @@ var MpoolReplaceCmd = &cli.Command{
 			if cctx.IsSet("fee-limit") {
 				maxFee, err := types.ParseFIL(cctx.String("fee-limit"))
 				if err != nil {
-					return fmt.Errorf("parsing max-spend: %w", err)
+					return xerrors.Errorf("parsing max-spend: %w", err)
 				}
 				mss = &lapi.MessageSendSpec{
 					MaxFee: abi.TokenAmount(maxFee),
@@ -472,7 +479,7 @@ var MpoolReplaceCmd = &cli.Command{
 			msg.GasPremium = abi.NewTokenAmount(0)
 			retm, err := api.GasEstimateMessageGas(ctx, &msg, mss, types.EmptyTSK)
 			if err != nil {
-				return fmt.Errorf("failed to estimate gas values: %w", err)
+				return xerrors.Errorf("failed to estimate gas values: %w", err)
 			}

 			msg.GasPremium = big.Max(retm.GasPremium, minRBF)
@@ -489,26 +496,26 @@ var MpoolReplaceCmd = &cli.Command{
 			}
 			msg.GasPremium, err = types.BigFromString(cctx.String("gas-premium"))
 			if err != nil {
-				return fmt.Errorf("parsing gas-premium: %w", err)
+				return xerrors.Errorf("parsing gas-premium: %w", err)
 			}
 			// TODO: estimate fee cap here
 			msg.GasFeeCap, err = types.BigFromString(cctx.String("gas-feecap"))
 			if err != nil {
-				return fmt.Errorf("parsing gas-feecap: %w", err)
+				return xerrors.Errorf("parsing gas-feecap: %w", err)
 			}
 		}

 		smsg, err := api.WalletSignMessage(ctx, msg.From, &msg)
 		if err != nil {
-			return fmt.Errorf("failed to sign message: %w", err)
+			return xerrors.Errorf("failed to sign message: %w", err)
 		}

 		cid, err := api.MpoolPush(ctx, smsg)
 		if err != nil {
-			return fmt.Errorf("failed to push new message to mempool: %w", err)
+			return xerrors.Errorf("failed to push new message to mempool: %w", err)
 		}

-		fmt.Println("new message cid: ", cid)
+		afmt.Println("new message cid: ", cid)
 		return nil
 	},
 }
@@ -531,6 +538,8 @@ var MpoolFindCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -548,7 +557,7 @@ var MpoolFindCmd = &cli.Command{
 		if cctx.IsSet("to") {
 			a, err := address.NewFromString(cctx.String("to"))
 			if err != nil {
-				return fmt.Errorf("'to' address was invalid: %w", err)
+				return xerrors.Errorf("'to' address was invalid: %w", err)
 			}

 			toFilter = a
@@ -557,7 +566,7 @@ var MpoolFindCmd = &cli.Command{
 		if cctx.IsSet("from") {
 			a, err := address.NewFromString(cctx.String("from"))
 			if err != nil {
-				return fmt.Errorf("'from' address was invalid: %w", err)
+				return xerrors.Errorf("'from' address was invalid: %w", err)
 			}

 			fromFilter = a
@@ -591,7 +600,7 @@ var MpoolFindCmd = &cli.Command{
 			return err
 		}

-		fmt.Println(string(b))
+		afmt.Println(string(b))
 		return nil
 	},
 }
@@ -605,6 +614,8 @@ var MpoolConfig = &cli.Command{
 			return cli.ShowCommandHelp(cctx, cctx.Command.Name)
 		}

+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -624,7 +635,7 @@ var MpoolConfig = &cli.Command{
 				return err
 			}

-			fmt.Println(string(bytes))
+			afmt.Println(string(bytes))
 		} else {
 			cfg := new(types.MpoolConfig)
 			bytes := []byte(cctx.Args().Get(0))
@@ -651,6 +662,8 @@ var MpoolGasPerfCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -717,7 +730,7 @@ var MpoolGasPerfCmd = &cli.Command{
 			gasReward := getGasReward(m)
 			gasPerf := getGasPerf(gasReward, m.Message.GasLimit)

-			fmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
+			afmt.Printf("%s\t%d\t%s\t%f\n", m.Message.From, m.Message.Nonce, gasReward, gasPerf)
 		}

 		return nil
@@ -0,0 +1,582 @@
+//stm: #cli
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"encoding/json"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStat(t *testing.T) {
+
+	t.Run("local", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// mock actor to return for the sender
+		actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr, toAddr}, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_STAT_002
+		err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1", "--local"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "Nonce past: 1")
+	})
+
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolStat))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// mock actor to return for the sender
+		actor := types.Actor{Nonce: 2, Balance: big.NewInt(200000)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+			mockApi.EXPECT().ChainGetTipSet(ctx, head.Parents()).Return(first, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().StateGetActor(ctx, senderAddr, head.Key()).Return(&actor, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_STAT_001
+		err = app.Run([]string{"mpool", "stat", "--basefee-lookback", "1"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "Nonce past: 1")
+	})
+}
+
+func TestPending(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_001
+		err = app.Run([]string{"mpool", "pending", "--cids"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("local", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_002
+		err = app.Run([]string{"mpool", "pending", "--local"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("to", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_003
+		err = app.Run([]string{"mpool", "pending", "--to", sm.Message.To.String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("from", func(t *testing.T) {
+
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolPending))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_PENDING_004
+		err = app.Run([]string{"mpool", "pending", "--from", sm.Message.From.String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+}
+
+func TestReplace(t *testing.T) {
+	t.Run("manual", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, &sm.Message).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_002
+		err = app.Run([]string{"mpool", "replace", "--gas-premium", "1", "--gas-feecap", "100", sm.Cid().String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("auto", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// gas fee param should be equal to the one passed in the cli invocation (used below)
+		maxFee := "1000000"
+		parsedFee, err := types.ParseFIL(maxFee)
+		if err != nil {
+			t.Fatal(err)
+		}
+		mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetMessage(ctx, sm.Cid()).Return(&sm.Message, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			// use gomock.any to match the message in expected api calls
+			// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
+			mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_002
+		err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Cid().String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("sender / nonce", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolReplaceCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		// gas fee param should be equal to the one passed in the cli invocation (used below)
+		maxFee := "1000000"
+		parsedFee, err := types.ParseFIL(maxFee)
+		if err != nil {
+			t.Fatal(err)
+		}
+		mss := api.MessageSendSpec{MaxFee: abi.TokenAmount(parsedFee)}
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainHead(ctx).Return(nil, nil),
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			// use gomock.any to match the message in expected api calls
+			// since the replace function modifies the message between calls, it would be pointless to try to match the exact argument
+			mockApi.EXPECT().GasEstimateMessageGas(ctx, gomock.Any(), &mss, types.EmptyTSK).Return(&sm.Message, nil),
+			mockApi.EXPECT().WalletSignMessage(ctx, sm.Message.From, gomock.Any()).Return(sm, nil),
+			mockApi.EXPECT().MpoolPush(ctx, sm).Return(sm.Cid(), nil),
+		)
+
+		//stm: @CLI_MEMPOOL_REPLACE_001
+		err = app.Run([]string{"mpool", "replace", "--auto", "--fee-limit", maxFee, sm.Message.From.String(), fmt.Sprint(sm.Message.Nonce)})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+}
+
+func TestFindMsg(t *testing.T) {
+	t.Run("from", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_001
+		err = app.Run([]string{"mpool", "find", "--from", sm.Message.From.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("to", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_002
+		err = app.Run([]string{"mpool", "find", "--to", sm.Message.To.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+
+	t.Run("method", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolFindCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 1, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_FIND_003
+		err = app.Run([]string{"mpool", "find", "--method", sm.Message.Method.String()})
+
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), sm.Cid().String())
+	})
+}
+
+func TestGasPerf(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 13, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_GAS_PERF_002
+		err = app.Run([]string{"mpool", "gas-perf", "--all", "true"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Message.From.String())
+		assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
+	})
+
+	t.Run("local", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolGasPerfCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// add blocks to the chain
+		first := mock.TipSet(mock.MkBlock(nil, 5, 4))
+		head := mock.TipSet(mock.MkBlock(first, 15, 7))
+
+		// create a signed message to be returned as a pending message
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		toAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		sm := mock.MkMessage(senderAddr, toAddr, 13, w)
+
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolPending(ctx, types.EmptyTSK).Return([]*types.SignedMessage{sm}, nil),
+			mockApi.EXPECT().WalletList(ctx).Return([]address.Address{senderAddr}, nil),
+			mockApi.EXPECT().ChainHead(ctx).Return(head, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_GAS_PERF_001
+		err = app.Run([]string{"mpool", "gas-perf"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), sm.Message.From.String())
+		assert.Contains(t, buf.String(), fmt.Sprint(sm.Message.Nonce))
+	})
+}
+
+func TestConfig(t *testing.T) {
+	t.Run("get", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 1234567, SizeLimitLow: 6, ReplaceByFeeRatio: 0.25}
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolGetConfig(ctx).Return(mpoolCfg, nil),
+		)
+
+		//stm: @CLI_MEMPOOL_CONFIG_001
+		err = app.Run([]string{"mpool", "config"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), mpoolCfg.PriorityAddrs[0].String())
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitHigh))
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.SizeLimitLow))
+		assert.Contains(t, buf.String(), fmt.Sprint(mpoolCfg.ReplaceByFeeRatio))
+	})
+
+	t.Run("set", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("mpool", MpoolConfig))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+		senderAddr, err := w.WalletNew(context.Background(), types.KTSecp256k1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		mpoolCfg := &types.MpoolConfig{PriorityAddrs: []address.Address{senderAddr}, SizeLimitHigh: 234567, SizeLimitLow: 3, ReplaceByFeeRatio: 0.33}
+		gomock.InOrder(
+			mockApi.EXPECT().MpoolSetConfig(ctx, mpoolCfg).Return(nil),
+		)
+
+		bytes, err := json.Marshal(mpoolCfg)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		//stm: @CLI_MEMPOOL_CONFIG_002
+		err = app.Run([]string{"mpool", "config", string(bytes)})
+		assert.NoError(t, err)
+	})
+}
@@ -36,6 +36,11 @@ var NetCmd = &cli.Command{
 		NetReachability,
 		NetBandwidthCmd,
 		NetBlockCmd,
+		NetStatCmd,
+		NetLimitCmd,
+		NetProtectAdd,
+		NetProtectRemove,
+		NetProtectList,
 	},
 }

@@ -606,3 +611,208 @@ var NetBlockListCmd = &cli.Command{
 		return nil
 	},
 }
+
+var NetStatCmd = &cli.Command{
+	Name:      "stat",
+	Usage:     "Report resource usage for a scope",
+	ArgsUsage: "scope",
+	Description: `Report resource usage for a scope.
+
+  The scope can be one of the following:
+  - system        -- reports the system aggregate resource usage.
+  - transient     -- reports the transient resource usage.
+  - svc:<service> -- reports the resource usage of a specific service.
+  - proto:<proto> -- reports the resource usage of a specific protocol.
+  - peer:<peer>   -- reports the resource usage of a specific peer.
+  - all           -- reports the resource usage for all currently active scopes.
+`,
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		args := cctx.Args().Slice()
+		if len(args) != 1 {
+			return xerrors.Errorf("must specify exactly one scope")
+		}
+		scope := args[0]
+
+		result, err := api.NetStat(ctx, scope)
+		if err != nil {
+			return err
+		}
+
+		enc := json.NewEncoder(os.Stdout)
+		return enc.Encode(result)
+	},
+}
+
+var NetLimitCmd = &cli.Command{
+	Name:      "limit",
+	Usage:     "Get or set resource limits for a scope",
+	ArgsUsage: "scope [limit]",
+	Description: `Get or set resource limits for a scope.
+
+  The scope can be one of the following:
+  - system        -- reports the system aggregate resource usage.
+  - transient     -- reports the transient resource usage.
+  - svc:<service> -- reports the resource usage of a specific service.
+  - proto:<proto> -- reports the resource usage of a specific protocol.
+  - peer:<peer>   -- reports the resource usage of a specific peer.
+
+ The limit is json-formatted, with the same structure as the limits file.
+`,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "set",
+			Usage: "set the limit for a scope",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		args := cctx.Args().Slice()
+
+		if cctx.Bool("set") {
+			if len(args) != 2 {
+				return xerrors.Errorf("must specify exactly a scope and a limit")
+			}
+			scope := args[0]
+			limitStr := args[1]
+
+			var limit atypes.NetLimit
+			err := json.Unmarshal([]byte(limitStr), &limit)
+			if err != nil {
+				return xerrors.Errorf("error decoding limit: %w", err)
+			}
+
+			return api.NetSetLimit(ctx, scope, limit)
+
+		}
+
+		if len(args) != 1 {
+			return xerrors.Errorf("must specify exactly one scope")
+		}
+		scope := args[0]
+
+		result, err := api.NetLimit(ctx, scope)
+		if err != nil {
+			return err
+		}
+
+		enc := json.NewEncoder(os.Stdout)
+		return enc.Encode(result)
+	},
+}
+
+var NetProtectAdd = &cli.Command{
+	Name:      "protect",
+	Usage:     "Add one or more peer IDs to the list of protected peer connections",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectAdd(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("added to protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+var NetProtectRemove = &cli.Command{
+	Name:      "unprotect",
+	Usage:     "Remove one or more peer IDs from the list of protected peer connections.",
+	ArgsUsage: "<peer-id> [<peer-id>...]",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		pids, err := decodePeerIDsFromArgs(cctx)
+		if err != nil {
+			return err
+		}
+
+		err = api.NetProtectRemove(ctx, pids)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("removed from protected peers:")
+		for _, pid := range pids {
+			fmt.Printf(" %s\n", pid)
+		}
+		return nil
+	},
+}
+
+// decodePeerIDsFromArgs decodes all the arguments present in cli.Context.Args as peer.ID.
+//
+// This function requires at least one argument to be present, and arguments must not be empty
+// string. Otherwise, an error is returned.
+func decodePeerIDsFromArgs(cctx *cli.Context) ([]peer.ID, error) {
+	pidArgs := cctx.Args().Slice()
+	if len(pidArgs) == 0 {
+		return nil, xerrors.Errorf("must specify at least one peer ID as an argument")
+	}
+	var pids []peer.ID
+	for _, pidStr := range pidArgs {
+		if pidStr == "" {
+			return nil, xerrors.Errorf("peer ID must not be empty")
+		}
+		pid, err := peer.Decode(pidStr)
+		if err != nil {
+			return nil, err
+		}
+		pids = append(pids, pid)
+	}
+	return pids, nil
+}
+
+var NetProtectList = &cli.Command{
+	Name:  "list-protected",
+	Usage: "List the peer IDs with protected connection.",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+		pids, err := api.NetProtectList(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, pid := range pids {
+			fmt.Printf("%s\n", pid)
+		}
+		return nil
+	},
+}
@@ -33,6 +33,8 @@ var SyncStatusCmd = &cli.Command{
 	Name:  "status",
 	Usage: "check sync status",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		apic, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -45,9 +47,9 @@ var SyncStatusCmd = &cli.Command{
 			return err
 		}

-		fmt.Println("sync status:")
+		afmt.Println("sync status:")
 		for _, ss := range state.ActiveSyncs {
-			fmt.Printf("worker %d:\n", ss.WorkerID)
+			afmt.Printf("worker %d:\n", ss.WorkerID)
 			var base, target []cid.Cid
 			var heightDiff int64
 			var theight abi.ChainEpoch
@@ -62,20 +64,20 @@ var SyncStatusCmd = &cli.Command{
 			} else {
 				heightDiff = 0
 			}
-			fmt.Printf("\tBase:\t%s\n", base)
-			fmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
-			fmt.Printf("\tHeight diff:\t%d\n", heightDiff)
-			fmt.Printf("\tStage: %s\n", ss.Stage)
-			fmt.Printf("\tHeight: %d\n", ss.Height)
+			afmt.Printf("\tBase:\t%s\n", base)
+			afmt.Printf("\tTarget:\t%s (%d)\n", target, theight)
+			afmt.Printf("\tHeight diff:\t%d\n", heightDiff)
+			afmt.Printf("\tStage: %s\n", ss.Stage)
+			afmt.Printf("\tHeight: %d\n", ss.Height)
 			if ss.End.IsZero() {
 				if !ss.Start.IsZero() {
-					fmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
+					afmt.Printf("\tElapsed: %s\n", time.Since(ss.Start))
 				}
 			} else {
-				fmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
+				afmt.Printf("\tElapsed: %s\n", ss.End.Sub(ss.Start))
 			}
 			if ss.Stage == api.StageSyncErrored {
-				fmt.Printf("\tError: %s\n", ss.Message)
+				afmt.Printf("\tError: %s\n", ss.Message)
 			}
 		}
 		return nil
@@ -168,6 +170,8 @@ var SyncCheckBadCmd = &cli.Command{
 	Usage:     "check if the given block was marked bad, and for what reason",
 	ArgsUsage: "[blockCid]",
 	Action: func(cctx *cli.Context) error {
+		afmt := NewAppFmt(cctx.App)
+
 		napi, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -190,11 +194,11 @@ var SyncCheckBadCmd = &cli.Command{
 		}

 		if reason == "" {
-			fmt.Println("block was not marked as bad")
+			afmt.Println("block was not marked as bad")
 			return nil
 		}

-		fmt.Println(reason)
+		afmt.Println(reason)
 		return nil
 	},
 }
@@ -0,0 +1,189 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSyncStatus(t *testing.T) {
+	app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncStatusCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ts1 := mock.TipSet(mock.MkBlock(nil, 0, 0))
+	ts2 := mock.TipSet(mock.MkBlock(ts1, 0, 0))
+
+	start := time.Now()
+	end := start.Add(time.Minute)
+
+	state := &api.SyncState{
+		ActiveSyncs: []api.ActiveSync{{
+			WorkerID: 1,
+			Base:     ts1,
+			Target:   ts2,
+			Stage:    api.StageMessages,
+			Height:   abi.ChainEpoch(0),
+			Start:    start,
+			End:      end,
+			Message:  "whatever",
+		}},
+		VMApplied: 0,
+	}
+
+	mockApi.EXPECT().SyncState(ctx).Return(state, nil)
+
+	//stm: @CLI_SYNC_STATUS_001
+	err := app.Run([]string{"sync", "status"})
+	assert.NoError(t, err)
+
+	out := buf.String()
+
+	// output is plaintext, had to do string matching
+	assert.Contains(t, out, fmt.Sprintf("Base:\t[%s]", ts1.Blocks()[0].Cid().String()))
+	assert.Contains(t, out, fmt.Sprintf("Target:\t[%s]", ts2.Blocks()[0].Cid().String()))
+	assert.Contains(t, out, "Height diff:\t1")
+	assert.Contains(t, out, "Stage: message sync")
+	assert.Contains(t, out, "Height: 0")
+	assert.Contains(t, out, "Elapsed: 1m0s")
+}
+
+func TestSyncMarkBad(t *testing.T) {
+	app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncMarkBadCmd))
+	defer done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	blk := mock.MkBlock(nil, 0, 0)
+
+	mockApi.EXPECT().SyncMarkBad(ctx, blk.Cid()).Return(nil)
+
+	//stm: @CLI_SYNC_MARK_BAD_001
+	err := app.Run([]string{"sync", "mark-bad", blk.Cid().String()})
+	assert.NoError(t, err)
+}
+
+func TestSyncUnmarkBad(t *testing.T) {
+	t.Run("one-block", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+
+		mockApi.EXPECT().SyncUnmarkBad(ctx, blk.Cid()).Return(nil)
+
+		//stm: @CLI_SYNC_UNMARK_BAD_001
+		err := app.Run([]string{"sync", "unmark-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+	})
+
+	t.Run("all", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncUnmarkBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		mockApi.EXPECT().SyncUnmarkAllBad(ctx).Return(nil)
+
+		//stm: @CLI_SYNC_UNMARK_BAD_002
+		err := app.Run([]string{"sync", "unmark-bad", "-all"})
+		assert.NoError(t, err)
+	})
+}
+
+func TestSyncCheckBad(t *testing.T) {
+	t.Run("not-bad", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+
+		mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return("", nil)
+
+		//stm: @CLI_SYNC_CHECK_BAD_002
+		err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), "block was not marked as bad")
+	})
+
+	t.Run("bad", func(t *testing.T) {
+		app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckBadCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+		reason := "whatever"
+
+		mockApi.EXPECT().SyncCheckBad(ctx, blk.Cid()).Return(reason, nil)
+
+		//stm: @CLI_SYNC_CHECK_BAD_001
+		err := app.Run([]string{"sync", "check-bad", blk.Cid().String()})
+		assert.NoError(t, err)
+
+		assert.Contains(t, buf.String(), reason)
+	})
+}
+
+func TestSyncCheckpoint(t *testing.T) {
+	t.Run("tipset", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		blk := mock.MkBlock(nil, 0, 0)
+		ts := mock.TipSet(blk)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetBlock(ctx, blk.Cid()).Return(blk, nil),
+			mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_SYNC_CHECKPOINT_001
+		err := app.Run([]string{"sync", "checkpoint", blk.Cid().String()})
+		assert.NoError(t, err)
+	})
+
+	t.Run("epoch", func(t *testing.T) {
+		app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("sync", SyncCheckpointCmd))
+		defer done()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		epoch := abi.ChainEpoch(0)
+		blk := mock.MkBlock(nil, 0, 0)
+		ts := mock.TipSet(blk)
+
+		gomock.InOrder(
+			mockApi.EXPECT().ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK).Return(ts, nil),
+			mockApi.EXPECT().SyncCheckpoint(ctx, ts.Key()).Return(nil),
+		)
+
+		//stm: @CLI_SYNC_CHECKPOINT_002
+		err := app.Run([]string{"sync", "checkpoint", fmt.Sprintf("-epoch=%d", epoch)})
+		assert.NoError(t, err)
+	})
+}
@@ -223,6 +223,11 @@ func GetCommonAPI(ctx *cli.Context) (api.CommonNet, jsonrpc.ClientCloser, error)
 }

 func GetFullNodeAPI(ctx *cli.Context) (v0api.FullNode, jsonrpc.ClientCloser, error) {
+	// use the mocked API in CLI unit tests, see cli/mocks_test.go for mock definition
+	if mock, ok := ctx.App.Metadata["test-full-api"]; ok {
+		return &v0api.WrapperV1Full{FullNode: mock.(v1api.FullNode)}, func() {}, nil
+	}
+
 	if tn, ok := ctx.App.Metadata["testnode-full"]; ok {
 		return &v0api.WrapperV1Full{FullNode: tn.(v1api.FullNode)}, func() {}, nil
 	}
@@ -276,6 +276,13 @@ var sealBenchCmd = &cli.Command{
 			if err != nil {
 				return xerrors.Errorf("failed to run seals: %w", err)
 			}
+			for _, s := range extendedSealedSectors {
+				sealedSectors = append(sealedSectors, proof.SectorInfo{
+					SealedCID:    s.SealedCID,
+					SectorNumber: s.SectorNumber,
+					SealProof:    s.SealProof,
+				})
+			}
 		} else {
 			// TODO: implement sbfs.List() and use that for all cases (preexisting sectorbuilder or not)

@@ -5,6 +5,7 @@ import (
 	"os"

 	"github.com/fatih/color"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"

 	"github.com/filecoin-project/lotus/api"
@@ -21,6 +22,7 @@ var dagstoreCmd = &cli.Command{
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
 		dagstoreGcCmd,
+		dagstoreLookupPiecesCmd,
 	},
 }

@@ -52,38 +54,7 @@ var dagstoreListShardsCmd = &cli.Command{
 			return err
 		}

-		if len(shards) == 0 {
-			return nil
-		}
-
-		tw := tablewriter.New(
-			tablewriter.Col("Key"),
-			tablewriter.Col("State"),
-			tablewriter.Col("Error"),
-		)
-
-		colors := map[string]color.Attribute{
-			"ShardStateAvailable": color.FgGreen,
-			"ShardStateServing":   color.FgBlue,
-			"ShardStateErrored":   color.FgRed,
-			"ShardStateNew":       color.FgYellow,
-		}
-
-		for _, s := range shards {
-			m := map[string]interface{}{
-				"Key": s.Key,
-				"State": func() string {
-					if c, ok := colors[s.State]; ok {
-						return color.New(c).Sprint(s.State)
-					}
-					return s.State
-				}(),
-				"Error": s.Error,
-			}
-			tw.Write(m)
-		}
-
-		return tw.Flush(os.Stdout)
+		return printTableShards(shards)
 	},
 }

@@ -265,3 +236,80 @@ var dagstoreGcCmd = &cli.Command{
 		return nil
 	},
 }
+
+func printTableShards(shards []api.DagstoreShardInfo) error {
+	if len(shards) == 0 {
+		return nil
+	}
+
+	tw := tablewriter.New(
+		tablewriter.Col("Key"),
+		tablewriter.Col("State"),
+		tablewriter.Col("Error"),
+	)
+
+	colors := map[string]color.Attribute{
+		"ShardStateAvailable": color.FgGreen,
+		"ShardStateServing":   color.FgBlue,
+		"ShardStateErrored":   color.FgRed,
+		"ShardStateNew":       color.FgYellow,
+	}
+
+	for _, s := range shards {
+		m := map[string]interface{}{
+			"Key": s.Key,
+			"State": func() string {
+				if c, ok := colors[s.State]; ok {
+					return color.New(c).Sprint(s.State)
+				}
+				return s.State
+			}(),
+			"Error": s.Error,
+		}
+		tw.Write(m)
+	}
+	return tw.Flush(os.Stdout)
+}
+
+var dagstoreLookupPiecesCmd = &cli.Command{
+	Name:      "lookup-pieces",
+	Usage:     "Lookup pieces that a given CID belongs to",
+	ArgsUsage: "<cid>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide a CID")
+		}
+
+		cidStr := cctx.Args().First()
+		cid, err := cid.Parse(cidStr)
+		if err != nil {
+			return fmt.Errorf("invalid CID: %w", err)
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shards, err := marketsApi.DagstoreLookupPieces(ctx, cid)
+		if err != nil {
+			return err
+		}
+
+		return printTableShards(shards)
+	},
+}
--- a/Show More
+++ b/Show More