Merge pull request #4787 from filecoin-project/refactor/fund-mgr-integ

refactor: integrate new FundManager

.circleci/config.yml

@@ -34,7 +34,7 @@ commands:
           condition: << parameters.linux >>
           steps:
             - run: sudo apt-get update
-            - run: sudo apt-get install ocl-icd-opencl-dev
+            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -276,7 +276,7 @@ jobs:
       - run: cd extern/filecoin-ffi && make
       - run:
           name: "replace lotus, filecoin-ffi, blst and fil-blst deps"
-          command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../fil-blst/blst && go mod edit -replace github.com/filecoin-project/fil-blst=../../fil-blst
+          command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../blst
       - run:
           name: "build lotus-soup testplan"
           command: pushd extern/oni/lotus-soup && go build -tags=testground .

.gitmodules

@@ -1,16 +1,15 @@
 [submodule "extern/filecoin-ffi"]
 	path = extern/filecoin-ffi
 	url = https://github.com/filecoin-project/filecoin-ffi.git
-	branch = master
 [submodule "extern/serialization-vectors"]
 	path = extern/serialization-vectors
-	url = https://github.com/filecoin-project/serialization-vectors
+	url = https://github.com/filecoin-project/serialization-vectors.git
 [submodule "extern/test-vectors"]
 	path = extern/test-vectors
 	url = https://github.com/filecoin-project/test-vectors.git
-[submodule "extern/fil-blst"]
-	path = extern/fil-blst
-	url = https://github.com/filecoin-project/fil-blst.git
 [submodule "extern/oni"]
 	path = extern/oni
-	url = https://github.com/filecoin-project/oni
+	url = https://github.com/filecoin-project/oni.git
+[submodule "extern/blst"]
+	path = extern/blst
+	url = https://github.com/supranational/blst.git

Makefile

@@ -179,7 +179,7 @@ BINS+=lotus-bench
 
 lotus-stats:
 	rm -f lotus-stats
-	go build -o lotus-stats ./cmd/lotus-stats
+	go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
 	go run github.com/GeertJohan/go.rice/rice append --exec lotus-stats -i ./build
 .PHONY: lotus-stats
 BINS+=lotus-stats

api/api_common.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/google/uuid"
+
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
@@ -58,6 +60,9 @@ type Common interface {
 	// trigger graceful shutdown
 	Shutdown(context.Context) error
 
+	// Session returns a random UUID of api provider session
+	Session(context.Context) (uuid.UUID, error)
+
 	Closing(context.Context) (<-chan struct{}, error)
 }
 

api/api_full.go

@@ -301,6 +301,8 @@ type FullNode interface {
 	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
 	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error)
+	// ClientCalcCommP calculates the CommP and data size of the specified CID
+	ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error)
 	// ClientCalcCommP calculates the CommP for a specified file
 	ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error)
 	// ClientGenCar generates a CAR file for the specified file.
@@ -312,6 +314,8 @@ type FullNode interface {
 	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
 	// ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
 	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
+	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
 	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
 	// which are stuck due to insufficient funds
 	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error
@@ -340,6 +344,8 @@ type FullNode interface {
 	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
 	// StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height.
 	StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)
+	// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
+	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
 
 	// StateNetworkName returns the name of the network the node is synced to
 	StateNetworkName(context.Context) (dtypes.NetworkName, error)
@@ -370,6 +376,8 @@ type FullNode interface {
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
 	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
+	// StateMinerSectorAllocated checks if a sector is allocated
+	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error)
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
 	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
 	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
@@ -506,8 +514,10 @@ type FullNode interface {
 	// along with the address removal.
 	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error)
 
-	MarketEnsureAvailable(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error)
-	// MarketFreeBalance
+	// MarketReserveFunds reserves funds for a deal
+	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error)
+	// MarketReleaseFunds releases funds reserved by MarketReserveFunds
+	MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error
 
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels
@@ -781,8 +791,9 @@ type IpldObject struct {
 }
 
 type ActiveSync struct {
-	Base   *types.TipSet
-	Target *types.TipSet
+	WorkerID uint64
+	Base     *types.TipSet
+	Target   *types.TipSet
 
 	Stage  SyncStateStage
 	Height abi.ChainEpoch
@@ -887,6 +898,12 @@ type DataSize struct {
 	PieceSize   abi.PaddedPieceSize
 }
 
+type DataCIDSize struct {
+	PayloadSize int64
+	PieceSize   abi.PaddedPieceSize
+	PieceCID    cid.Cid
+}
+
 type CommPRet struct {
 	Root cid.Cid
 	Size abi.UnpaddedPieceSize

api/api_storage.go

@@ -5,7 +5,10 @@ import (
 	"context"
 	"time"
 
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
@@ -62,11 +65,12 @@ type StorageMiner interface {
 
 	// WorkerConnect tells the node to connect to workers RPC
 	WorkerConnect(context.Context, string) error
-	WorkerStats(context.Context) (map[uint64]storiface.WorkerStats, error)
-	WorkerJobs(context.Context) (map[uint64][]storiface.WorkerJob, error)
+	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error)
+	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)
+	storiface.WorkerReturn
 
 	// SealingSchedDiag dumps internal sealing scheduler state
-	SealingSchedDiag(context.Context) (interface{}, error)
+	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error)
 
 	stores.SectorIndex
 
@@ -81,6 +85,10 @@ type StorageMiner interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
 	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
 	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
+	// MinerRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
+	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
+	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
 
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
 	DealsList(ctx context.Context) ([]MarketDeal, error)

api/api_worker.go

@@ -2,15 +2,13 @@ package api
 
 import (
 	"context"
-	"io"
 
-	"github.com/ipfs/go-cid"
+	"github.com/google/uuid"
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/build"
 )
@@ -23,18 +21,26 @@ type WorkerAPI interface {
 	Paths(context.Context) ([]stores.StoragePath, error)
 	Info(context.Context) (storiface.WorkerInfo, error)
 
-	AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)
+	storiface.WorkerCalls
 
-	storage.Sealer
-
-	MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error
-
-	UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error
-	ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)
+	// Storage / Other
+	Remove(ctx context.Context, sector abi.SectorID) error
 
 	StorageAddLocal(ctx context.Context, path string) error
 
-	Fetch(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error
+	// SetEnabled marks the worker as enabled/disabled. Not that this setting
+	// may take a few seconds to propagate to task scheduler
+	SetEnabled(ctx context.Context, enabled bool) error
 
-	Closing(context.Context) (<-chan struct{}, error)
+	Enabled(ctx context.Context) (bool, error)
+
+	// WaitQuiet blocks until there are no tasks running
+	WaitQuiet(ctx context.Context) error
+
+	// returns a random UUID of worker session, generated randomly when worker
+	// process starts
+	ProcessSession(context.Context) (uuid.UUID, error)
+
+	// Like ProcessSession, but returns an error when worker is disabled
+	Session(context.Context) (uuid.UUID, error)
 }

api/apistruct/struct.go

@@ -5,8 +5,7 @@ import (
 	"io"
 	"time"
 
-	stnetwork "github.com/filecoin-project/go-state-types/network"
-
+	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	metrics "github.com/libp2p/go-libp2p-core/metrics"
 	"github.com/libp2p/go-libp2p-core/network"
@@ -25,6 +24,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
+	stnetwork "github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@@ -68,6 +68,7 @@ type CommonStruct struct {
 		LogSetLevel func(context.Context, string, string) error `perm:"write"`
 
 		Shutdown func(context.Context) error                    `perm:"admin"`
+		Session  func(context.Context) (uuid.UUID, error)       `perm:"read"`
 		Closing  func(context.Context) (<-chan struct{}, error) `perm:"read"`
 	}
 }
@@ -166,12 +167,14 @@ type FullNodeStruct struct {
 		ClientRetrieve                            func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error                                       `perm:"admin"`
 		ClientRetrieveWithEvents                  func(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) `perm:"admin"`
 		ClientQueryAsk                            func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error)                    `perm:"read"`
+		ClientDealPieceCID                        func(ctx context.Context, root cid.Cid) (api.DataCIDSize, error)                                                  `perm:"read"`
 		ClientCalcCommP                           func(ctx context.Context, inpath string) (*api.CommPRet, error)                                                   `perm:"read"`
 		ClientGenCar                              func(ctx context.Context, ref api.FileRef, outpath string) error                                                  `perm:"write"`
 		ClientDealSize                            func(ctx context.Context, root cid.Cid) (api.DataSize, error)                                                     `perm:"read"`
 		ClientListDataTransfers                   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                      `perm:"write"`
 		ClientDataTransferUpdates                 func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                 `perm:"write"`
 		ClientRestartDataTransfer                 func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error          `perm:"write"`
+		ClientCancelDataTransfer                  func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error          `perm:"write"`
 		ClientRetrieveTryRestartInsufficientFunds func(ctx context.Context, paymentChannel address.Address) error                                                   `perm:"write"`
 
 		StateNetworkName                   func(context.Context) (dtypes.NetworkName, error)                                                                   `perm:"read"`
@@ -188,6 +191,7 @@ type FullNodeStruct struct {
 		StateMinerPreCommitDepositForPower func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
 		StateMinerInitialPledgeCollateral  func(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)            `perm:"read"`
 		StateMinerAvailableBalance         func(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)                                       `perm:"read"`
+		StateMinerSectorAllocated          func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error)                             `perm:"read"`
 		StateSectorPreCommitInfo           func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
 		StateSectorGetInfo                 func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)         `perm:"read"`
 		StateSectorExpiration              func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error)          `perm:"read"`
@@ -211,6 +215,7 @@ type FullNodeStruct struct {
 		StateGetReceipt                    func(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)                                      `perm:"read"`
 		StateMinerSectorCount              func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)                                   `perm:"read"`
 		StateListMessages                  func(ctx context.Context, match *api.MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)     `perm:"read"`
+		StateDecodeParams                  func(context.Context, address.Address, abi.MethodNum, []byte, types.TipSetKey) (interface{}, error)                 `perm:"read"`
 		StateCompute                       func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error)           `perm:"read"`
 		StateVerifierStatus                func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error)                                  `perm:"read"`
 		StateVerifiedClientStatus          func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error)                                  `perm:"read"`
@@ -236,7 +241,8 @@ type FullNodeStruct struct {
 		MsigSwapCancel          func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error)                               `perm:"sign"`
 		MsigRemoveSigner        func(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error)              `perm:"sign"`
 
-		MarketEnsureAvailable func(context.Context, address.Address, address.Address, types.BigInt) (cid.Cid, error) `perm:"sign"`
+		MarketReserveFunds func(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) `perm:"sign"`
+		MarketReleaseFunds func(ctx context.Context, addr address.Address, amt types.BigInt) error                                    `perm:"sign"`
 
 		PaychGet                    func(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error)           `perm:"sign"`
 		PaychGetWaitReady           func(context.Context, cid.Cid) (address.Address, error)                                                   `perm:"sign"`
@@ -284,6 +290,8 @@ type StorageMinerStruct struct {
 		MarketGetRetrievalAsk     func(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                                      `perm:"read"`
 		MarketListDataTransfers   func(ctx context.Context) ([]api.DataTransferChannel, error)                                                                                                                 `perm:"write"`
 		MarketDataTransferUpdates func(ctx context.Context) (<-chan api.DataTransferChannel, error)                                                                                                            `perm:"write"`
+		MarketRestartDataTransfer func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                                     `perm:"read"`
+		MarketCancelDataTransfer  func(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                                     `perm:"read"`
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
@@ -299,24 +307,36 @@ type StorageMinerStruct struct {
 		SectorRemove                  func(context.Context, abi.SectorNumber) error                                                 `perm:"admin"`
 		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error                                          `perm:"admin"`
 
-		WorkerConnect func(context.Context, string) error                             `perm:"admin"` // TODO: worker perm
-		WorkerStats   func(context.Context) (map[uint64]storiface.WorkerStats, error) `perm:"admin"`
-		WorkerJobs    func(context.Context) (map[uint64][]storiface.WorkerJob, error) `perm:"admin"`
+		WorkerConnect func(context.Context, string) error                                `perm:"admin" retry:"true"` // TODO: worker perm
+		WorkerStats   func(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) `perm:"admin"`
+		WorkerJobs    func(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) `perm:"admin"`
 
-		SealingSchedDiag func(context.Context) (interface{}, error) `perm:"admin"`
+		ReturnAddPiece        func(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err string) error          `perm:"admin" retry:"true"`
+		ReturnSealPreCommit1  func(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err string) error `perm:"admin" retry:"true"`
+		ReturnSealPreCommit2  func(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err string) error `perm:"admin" retry:"true"`
+		ReturnSealCommit1     func(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err string) error    `perm:"admin" retry:"true"`
+		ReturnSealCommit2     func(ctx context.Context, callID storiface.CallID, proof storage.Proof, err string) error       `perm:"admin" retry:"true"`
+		ReturnFinalizeSector  func(ctx context.Context, callID storiface.CallID, err string) error                            `perm:"admin" retry:"true"`
+		ReturnReleaseUnsealed func(ctx context.Context, callID storiface.CallID, err string) error                            `perm:"admin" retry:"true"`
+		ReturnMoveStorage     func(ctx context.Context, callID storiface.CallID, err string) error                            `perm:"admin" retry:"true"`
+		ReturnUnsealPiece     func(ctx context.Context, callID storiface.CallID, err string) error                            `perm:"admin" retry:"true"`
+		ReturnReadPiece       func(ctx context.Context, callID storiface.CallID, ok bool, err string) error                   `perm:"admin" retry:"true"`
+		ReturnFetch           func(ctx context.Context, callID storiface.CallID, err string) error                            `perm:"admin" retry:"true"`
 
-		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                             `perm:"admin"`
-		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                    `perm:"admin"`
-		StorageStat          func(context.Context, stores.ID) (fsutil.FsStat, error)                                                                                `perm:"admin"`
-		StorageAttach        func(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                         `perm:"admin"`
-		StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error                                                      `perm:"admin"`
-		StorageDropSector    func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error                                                            `perm:"admin"`
-		StorageFindSector    func(context.Context, abi.SectorID, stores.SectorFileType, abi.SectorSize, bool) ([]stores.SectorStorageInfo, error)                   `perm:"admin"`
-		StorageInfo          func(context.Context, stores.ID) (stores.StorageInfo, error)                                                                           `perm:"admin"`
-		StorageBestAlloc     func(ctx context.Context, allocate stores.SectorFileType, ssize abi.SectorSize, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
-		StorageReportHealth  func(ctx context.Context, id stores.ID, report stores.HealthReport) error                                                              `perm:"admin"`
-		StorageLock          func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error                          `perm:"admin"`
-		StorageTryLock       func(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error)                  `perm:"admin"`
+		SealingSchedDiag func(context.Context, bool) (interface{}, error) `perm:"admin"`
+
+		StorageList          func(context.Context) (map[stores.ID][]stores.Decl, error)                                                                                   `perm:"admin"`
+		StorageLocal         func(context.Context) (map[stores.ID]string, error)                                                                                          `perm:"admin"`
+		StorageStat          func(context.Context, stores.ID) (fsutil.FsStat, error)                                                                                      `perm:"admin"`
+		StorageAttach        func(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                               `perm:"admin"`
+		StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, storiface.SectorFileType, bool) error                                                         `perm:"admin"`
+		StorageDropSector    func(context.Context, stores.ID, abi.SectorID, storiface.SectorFileType) error                                                               `perm:"admin"`
+		StorageFindSector    func(context.Context, abi.SectorID, storiface.SectorFileType, abi.SectorSize, bool) ([]stores.SectorStorageInfo, error)                      `perm:"admin"`
+		StorageInfo          func(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                 `perm:"admin"`
+		StorageBestAlloc     func(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, sealing storiface.PathType) ([]stores.StorageInfo, error) `perm:"admin"`
+		StorageReportHealth  func(ctx context.Context, id stores.ID, report stores.HealthReport) error                                                                    `perm:"admin"`
+		StorageLock          func(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                          `perm:"admin"`
+		StorageTryLock       func(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                  `perm:"admin"`
 
 		DealsImportData                       func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
 		DealsList                             func(ctx context.Context) ([]api.MarketDeal, error)               `perm:"read"`
@@ -352,23 +372,28 @@ type WorkerStruct struct {
 		Paths     func(context.Context) ([]stores.StoragePath, error)            `perm:"admin"`
 		Info      func(context.Context) (storiface.WorkerInfo, error)            `perm:"admin"`
 
-		AddPiece        func(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)                      `perm:"admin"`
-		SealPreCommit1  func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error)                                                           `perm:"admin"`
-		SealPreCommit2  func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error)                                                                                            `perm:"admin"`
-		SealCommit1     func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"`
-		SealCommit2     func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error)                                                                                                             `perm:"admin"`
-		FinalizeSector  func(context.Context, abi.SectorID, []storage.Range) error                                                                                                                                 `perm:"admin"`
-		ReleaseUnsealed func(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error                                                                                                           `perm:"admin"`
-		Remove          func(ctx context.Context, sector abi.SectorID) error                                                                                                                                       `perm:"admin"`
-		MoveStorage     func(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error                                                                                                          `perm:"admin"`
-		StorageAddLocal func(ctx context.Context, path string) error                                                                                                                                               `perm:"admin"`
+		AddPiece        func(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                 `perm:"admin"`
+		SealPreCommit1  func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                              `perm:"admin"`
+		SealPreCommit2  func(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                     `perm:"admin"`
+		SealCommit1     func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) `perm:"admin"`
+		SealCommit2     func(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         `perm:"admin"`
+		FinalizeSector  func(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                   `perm:"admin"`
+		ReleaseUnsealed func(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) (storiface.CallID, error)                                                                                     `perm:"admin"`
+		MoveStorage     func(ctx context.Context, sector abi.SectorID, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 `perm:"admin"`
+		UnsealPiece     func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           `perm:"admin"`
+		ReadPiece       func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (storiface.CallID, error)                                                             `perm:"admin"`
+		Fetch           func(context.Context, abi.SectorID, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                       `perm:"admin"`
 
-		UnsealPiece func(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error `perm:"admin"`
-		ReadPiece   func(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error)           `perm:"admin"`
+		Remove          func(ctx context.Context, sector abi.SectorID) error `perm:"admin"`
+		StorageAddLocal func(ctx context.Context, path string) error         `perm:"admin"`
 
-		Fetch func(context.Context, abi.SectorID, stores.SectorFileType, stores.PathType, stores.AcquireMode) error `perm:"admin"`
+		SetEnabled func(ctx context.Context, enabled bool) error `perm:"admin"`
+		Enabled    func(ctx context.Context) (bool, error)       `perm:"admin"`
 
-		Closing func(context.Context) (<-chan struct{}, error) `perm:"admin"`
+		WaitQuiet func(ctx context.Context) error `perm:"admin"`
+
+		ProcessSession func(context.Context) (uuid.UUID, error) `perm:"admin"`
+		Session        func(context.Context) (uuid.UUID, error) `perm:"admin"`
 	}
 }
 
@@ -495,6 +520,10 @@ func (c *CommonStruct) Shutdown(ctx context.Context) error {
 	return c.Internal.Shutdown(ctx)
 }
 
+func (c *CommonStruct) Session(ctx context.Context) (uuid.UUID, error) {
+	return c.Internal.Session(ctx)
+}
+
 func (c *CommonStruct) Closing(ctx context.Context) (<-chan struct{}, error) {
 	return c.Internal.Closing(ctx)
 }
@@ -556,6 +585,11 @@ func (c *FullNodeStruct) ClientRetrieveWithEvents(ctx context.Context, order api
 func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) {
 	return c.Internal.ClientQueryAsk(ctx, p, miner)
 }
+
+func (c *FullNodeStruct) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) {
+	return c.Internal.ClientDealPieceCID(ctx, root)
+}
+
 func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string) (*api.CommPRet, error) {
 	return c.Internal.ClientCalcCommP(ctx, inpath)
 }
@@ -580,6 +614,10 @@ func (c *FullNodeStruct) ClientRestartDataTransfer(ctx context.Context, transfer
 	return c.Internal.ClientRestartDataTransfer(ctx, transferID, otherPeer, isInitiator)
 }
 
+func (c *FullNodeStruct) ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.ClientCancelDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
 func (c *FullNodeStruct) ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error {
 	return c.Internal.ClientRetrieveTryRestartInsufficientFunds(ctx, paymentChannel)
 }
@@ -892,6 +930,10 @@ func (c *FullNodeStruct) StateMinerAvailableBalance(ctx context.Context, maddr a
 	return c.Internal.StateMinerAvailableBalance(ctx, maddr, tsk)
 }
 
+func (c *FullNodeStruct) StateMinerSectorAllocated(ctx context.Context, maddr address.Address, s abi.SectorNumber, tsk types.TipSetKey) (bool, error) {
+	return c.Internal.StateMinerSectorAllocated(ctx, maddr, s, tsk)
+}
+
 func (c *FullNodeStruct) StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) {
 	return c.Internal.StateSectorPreCommitInfo(ctx, maddr, n, tsk)
 }
@@ -980,6 +1022,10 @@ func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *api.Messa
 	return c.Internal.StateListMessages(ctx, match, tsk, toht)
 }
 
+func (c *FullNodeStruct) StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) {
+	return c.Internal.StateDecodeParams(ctx, toAddr, method, params, tsk)
+}
+
 func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch, msgs []*types.Message, tsk types.TipSetKey) (*api.ComputeStateOutput, error) {
 	return c.Internal.StateCompute(ctx, height, msgs, tsk)
 }
@@ -1072,8 +1118,12 @@ func (c *FullNodeStruct) MsigRemoveSigner(ctx context.Context, msig address.Addr
 	return c.Internal.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease)
 }
 
-func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
-	return c.Internal.MarketEnsureAvailable(ctx, addr, wallet, amt)
+func (c *FullNodeStruct) MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) {
+	return c.Internal.MarketReserveFunds(ctx, wallet, addr, amt)
+}
+
+func (c *FullNodeStruct) MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error {
+	return c.Internal.MarketReleaseFunds(ctx, addr, amt)
 }
 
 func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*api.ChannelInfo, error) {
@@ -1212,31 +1262,75 @@ func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) erro
 	return c.Internal.WorkerConnect(ctx, url)
 }
 
-func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]storiface.WorkerStats, error) {
+func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uuid.UUID]storiface.WorkerStats, error) {
 	return c.Internal.WorkerStats(ctx)
 }
 
-func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uint64][]storiface.WorkerJob, error) {
+func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) {
 	return c.Internal.WorkerJobs(ctx)
 }
 
-func (c *StorageMinerStruct) SealingSchedDiag(ctx context.Context) (interface{}, error) {
-	return c.Internal.SealingSchedDiag(ctx)
+func (c *StorageMinerStruct) ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err string) error {
+	return c.Internal.ReturnAddPiece(ctx, callID, pi, err)
+}
+
+func (c *StorageMinerStruct) ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err string) error {
+	return c.Internal.ReturnSealPreCommit1(ctx, callID, p1o, err)
+}
+
+func (c *StorageMinerStruct) ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err string) error {
+	return c.Internal.ReturnSealPreCommit2(ctx, callID, sealed, err)
+}
+
+func (c *StorageMinerStruct) ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err string) error {
+	return c.Internal.ReturnSealCommit1(ctx, callID, out, err)
+}
+
+func (c *StorageMinerStruct) ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err string) error {
+	return c.Internal.ReturnSealCommit2(ctx, callID, proof, err)
+}
+
+func (c *StorageMinerStruct) ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err string) error {
+	return c.Internal.ReturnFinalizeSector(ctx, callID, err)
+}
+
+func (c *StorageMinerStruct) ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err string) error {
+	return c.Internal.ReturnReleaseUnsealed(ctx, callID, err)
+}
+
+func (c *StorageMinerStruct) ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err string) error {
+	return c.Internal.ReturnMoveStorage(ctx, callID, err)
+}
+
+func (c *StorageMinerStruct) ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err string) error {
+	return c.Internal.ReturnUnsealPiece(ctx, callID, err)
+}
+
+func (c *StorageMinerStruct) ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err string) error {
+	return c.Internal.ReturnReadPiece(ctx, callID, ok, err)
+}
+
+func (c *StorageMinerStruct) ReturnFetch(ctx context.Context, callID storiface.CallID, err string) error {
+	return c.Internal.ReturnFetch(ctx, callID, err)
+}
+
+func (c *StorageMinerStruct) SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) {
+	return c.Internal.SealingSchedDiag(ctx, doSched)
 }
 
 func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st fsutil.FsStat) error {
 	return c.Internal.StorageAttach(ctx, si, st)
 }
 
-func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType, primary bool) error {
+func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error {
 	return c.Internal.StorageDeclareSector(ctx, storageId, s, ft, primary)
 }
 
-func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft stores.SectorFileType) error {
+func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft storiface.SectorFileType) error {
 	return c.Internal.StorageDropSector(ctx, storageId, s, ft)
 }
 
-func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) {
+func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) {
 	return c.Internal.StorageFindSector(ctx, si, types, ssize, allowFetch)
 }
 
@@ -1256,7 +1350,7 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto
 	return c.Internal.StorageInfo(ctx, id)
 }
 
-func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate stores.SectorFileType, ssize abi.SectorSize, pt stores.PathType) ([]stores.StorageInfo, error) {
+func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pt storiface.PathType) ([]stores.StorageInfo, error) {
 	return c.Internal.StorageBestAlloc(ctx, allocate, ssize, pt)
 }
 
@@ -1264,11 +1358,11 @@ func (c *StorageMinerStruct) StorageReportHealth(ctx context.Context, id stores.
 	return c.Internal.StorageReportHealth(ctx, id, report)
 }
 
-func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) error {
+func (c *StorageMinerStruct) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error {
 	return c.Internal.StorageLock(ctx, sector, read, write)
 }
 
-func (c *StorageMinerStruct) StorageTryLock(ctx context.Context, sector abi.SectorID, read stores.SectorFileType, write stores.SectorFileType) (bool, error) {
+func (c *StorageMinerStruct) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) {
 	return c.Internal.StorageTryLock(ctx, sector, read, write)
 }
 
@@ -1316,6 +1410,14 @@ func (c *StorageMinerStruct) MarketDataTransferUpdates(ctx context.Context) (<-c
 	return c.Internal.MarketDataTransferUpdates(ctx)
 }
 
+func (c *StorageMinerStruct) MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.MarketRestartDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
+func (c *StorageMinerStruct) MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error {
+	return c.Internal.MarketCancelDataTransfer(ctx, transferID, otherPeer, isInitiator)
+}
+
 func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error {
 	return c.Internal.DealsImportData(ctx, dealPropCid, file)
 }
@@ -1406,60 +1508,76 @@ func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
 	return w.Internal.Info(ctx)
 }
 
-func (w *WorkerStruct) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
+func (w *WorkerStruct) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error) {
 	return w.Internal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData)
 }
 
-func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
+func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error) {
 	return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces)
 }
 
-func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sector abi.SectorID, p1o storage.PreCommit1Out) (storage.SectorCids, error) {
-	return w.Internal.SealPreCommit2(ctx, sector, p1o)
+func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storiface.CallID, error) {
+	return w.Internal.SealPreCommit2(ctx, sector, pc1o)
 }
 
-func (w *WorkerStruct) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
+func (w *WorkerStruct) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) {
 	return w.Internal.SealCommit1(ctx, sector, ticket, seed, pieces, cids)
 }
 
-func (w *WorkerStruct) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) {
+func (w *WorkerStruct) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storiface.CallID, error) {
 	return w.Internal.SealCommit2(ctx, sector, c1o)
 }
 
-func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error {
+func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) (storiface.CallID, error) {
 	return w.Internal.FinalizeSector(ctx, sector, keepUnsealed)
 }
 
-func (w *WorkerStruct) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error {
+func (w *WorkerStruct) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) (storiface.CallID, error) {
 	return w.Internal.ReleaseUnsealed(ctx, sector, safeToFree)
 }
 
+func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID, types storiface.SectorFileType) (storiface.CallID, error) {
+	return w.Internal.MoveStorage(ctx, sector, types)
+}
+
+func (w *WorkerStruct) UnsealPiece(ctx context.Context, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, c cid.Cid) (storiface.CallID, error) {
+	return w.Internal.UnsealPiece(ctx, sector, offset, size, ticket, c)
+}
+
+func (w *WorkerStruct) ReadPiece(ctx context.Context, sink io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (storiface.CallID, error) {
+	return w.Internal.ReadPiece(ctx, sink, sector, offset, size)
+}
+
+func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) (storiface.CallID, error) {
+	return w.Internal.Fetch(ctx, id, fileType, ptype, am)
+}
+
 func (w *WorkerStruct) Remove(ctx context.Context, sector abi.SectorID) error {
 	return w.Internal.Remove(ctx, sector)
 }
 
-func (w *WorkerStruct) MoveStorage(ctx context.Context, sector abi.SectorID, types stores.SectorFileType) error {
-	return w.Internal.MoveStorage(ctx, sector, types)
-}
-
 func (w *WorkerStruct) StorageAddLocal(ctx context.Context, path string) error {
 	return w.Internal.StorageAddLocal(ctx, path)
 }
 
-func (w *WorkerStruct) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
-	return w.Internal.UnsealPiece(ctx, id, index, size, randomness, c)
+func (w *WorkerStruct) SetEnabled(ctx context.Context, enabled bool) error {
+	return w.Internal.SetEnabled(ctx, enabled)
 }
 
-func (w *WorkerStruct) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
-	return w.Internal.ReadPiece(ctx, writer, id, index, size)
+func (w *WorkerStruct) Enabled(ctx context.Context) (bool, error) {
+	return w.Internal.Enabled(ctx)
 }
 
-func (w *WorkerStruct) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error {
-	return w.Internal.Fetch(ctx, id, fileType, ptype, am)
+func (w *WorkerStruct) WaitQuiet(ctx context.Context) error {
+	return w.Internal.WaitQuiet(ctx)
 }
 
-func (w *WorkerStruct) Closing(ctx context.Context) (<-chan struct{}, error) {
-	return w.Internal.Closing(ctx)
+func (w *WorkerStruct) ProcessSession(ctx context.Context) (uuid.UUID, error) {
+	return w.Internal.ProcessSession(ctx)
+}
+
+func (w *WorkerStruct) Session(ctx context.Context) (uuid.UUID, error) {
+	return w.Internal.Session(ctx)
 }
 
 func (g GatewayStruct) ChainGetBlockMessages(ctx context.Context, c cid.Cid) (*api.BlockMessages, error) {

api/test/mining.go

@@ -58,14 +58,11 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 
 	newHeads, err := api.ChainNotify(ctx)
 	require.NoError(t, err)
-	initHead := (<-newHeads)[0]
-	if initHead.Val.Height() != 2 {
-		<-newHeads
-	}
+	at := (<-newHeads)[0].Val.Height()
 
 	h1, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(2), h1.Height())
+	require.Equal(t, int64(at), int64(h1.Height()))
 
 	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
@@ -74,16 +71,16 @@ func (ts *testSuite) testMiningReal(t *testing.T) {
 
 	h2, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(3), h2.Height())
+	require.Greater(t, int64(h2.Height()), int64(h1.Height()))
 
 	MineUntilBlock(ctx, t, apis[0], sn[0], nil)
 	require.NoError(t, err)
 
 	<-newHeads
 
-	h2, err = api.ChainHead(ctx)
+	h3, err := api.ChainHead(ctx)
 	require.NoError(t, err)
-	require.Equal(t, abi.ChainEpoch(4), h2.Height())
+	require.Greater(t, int64(h3.Height()), int64(h2.Height()))
 }
 
 func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {

build/params_2k.go

@@ -22,7 +22,7 @@ const UpgradeTapeHeight = -4
 var UpgradeActorsV2Height = abi.ChainEpoch(10)
 var UpgradeLiftoffHeight = abi.ChainEpoch(-5)
 
-const UpgradeKumquatHeight = -6
+const UpgradeKumquatHeight = 15
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,

build/version.go

@@ -84,8 +84,8 @@ func VersionForType(nodeType NodeType) (Version, error) {
 // semver versions of the rpc api exposed
 var (
 	FullAPIVersion   = newVer(0, 17, 0)
-	MinerAPIVersion  = newVer(0, 16, 0)
-	WorkerAPIVersion = newVer(0, 15, 0)
+	MinerAPIVersion  = newVer(0, 17, 0)
+	WorkerAPIVersion = newVer(0, 16, 0)
 )
 
 //nolint:varcheck,deadcode

chain/actors/builtin/market/market.go

@@ -81,6 +81,7 @@ type DealProposals interface {
 type PublishStorageDealsParams = market0.PublishStorageDealsParams
 type PublishStorageDealsReturn = market0.PublishStorageDealsReturn
 type VerifyDealsForActivationParams = market0.VerifyDealsForActivationParams
+type WithdrawBalanceParams = market0.WithdrawBalanceParams
 
 type ClientDealProposal = market0.ClientDealProposal
 

chain/actors/builtin/miner/miner.go

@@ -77,6 +77,7 @@ type State interface {
 	DeadlinesChanged(State) (bool, error)
 
 	Info() (MinerInfo, error)
+	MinerInfoChanged(State) (bool, error)
 
 	DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error)
 

chain/actors/builtin/miner/v0.go

@@ -277,6 +277,15 @@ func (s *state0) DeadlinesChanged(other State) (bool, error) {
 	return !s.State.Deadlines.Equals(other0.Deadlines), nil
 }
 
+func (s *state0) MinerInfoChanged(other State) (bool, error) {
+	other0, ok := other.(*state0)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Info.Equals(other0.State.Info), nil
+}
+
 func (s *state0) Info() (MinerInfo, error) {
 	info, err := s.State.GetInfo(s.store)
 	if err != nil {

chain/actors/builtin/miner/v2.go

@@ -276,6 +276,15 @@ func (s *state2) DeadlinesChanged(other State) (bool, error) {
 	return !s.State.Deadlines.Equals(other2.Deadlines), nil
 }
 
+func (s *state2) MinerInfoChanged(other State) (bool, error) {
+	other0, ok := other.(*state2)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Info.Equals(other0.State.Info), nil
+}
+
 func (s *state2) Info() (MinerInfo, error) {
 	info, err := s.State.GetInfo(s.store)
 	if err != nil {

chain/actors/builtin/multisig/diff.go

@@ -0,0 +1,134 @@
+package multisig
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+)
+
+type PendingTransactionChanges struct {
+	Added    []TransactionChange
+	Modified []TransactionModification
+	Removed  []TransactionChange
+}
+
+type TransactionChange struct {
+	TxID int64
+	Tx   Transaction
+}
+
+type TransactionModification struct {
+	TxID int64
+	From Transaction
+	To   Transaction
+}
+
+func DiffPendingTransactions(pre, cur State) (*PendingTransactionChanges, error) {
+	results := new(PendingTransactionChanges)
+	if changed, err := pre.PendingTxnChanged(cur); err != nil {
+		return nil, err
+	} else if !changed { // if nothing has changed then return an empty result and bail.
+		return results, nil
+	}
+
+	pret, err := pre.transactions()
+	if err != nil {
+		return nil, err
+	}
+
+	curt, err := cur.transactions()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := adt.DiffAdtMap(pret, curt, &transactionDiffer{results, pre, cur}); err != nil {
+		return nil, err
+	}
+	return results, nil
+}
+
+type transactionDiffer struct {
+	Results    *PendingTransactionChanges
+	pre, after State
+}
+
+func (t *transactionDiffer) AsKey(key string) (abi.Keyer, error) {
+	txID, err := abi.ParseIntKey(key)
+	if err != nil {
+		return nil, err
+	}
+	return abi.IntKey(txID), nil
+}
+
+func (t *transactionDiffer) Add(key string, val *cbg.Deferred) error {
+	txID, err := abi.ParseIntKey(key)
+	if err != nil {
+		return err
+	}
+	tx, err := t.after.decodeTransaction(val)
+	if err != nil {
+		return err
+	}
+	t.Results.Added = append(t.Results.Added, TransactionChange{
+		TxID: txID,
+		Tx:   tx,
+	})
+	return nil
+}
+
+func (t *transactionDiffer) Modify(key string, from, to *cbg.Deferred) error {
+	txID, err := abi.ParseIntKey(key)
+	if err != nil {
+		return err
+	}
+
+	txFrom, err := t.pre.decodeTransaction(from)
+	if err != nil {
+		return err
+	}
+
+	txTo, err := t.after.decodeTransaction(to)
+	if err != nil {
+		return err
+	}
+
+	if approvalsChanged(txFrom.Approved, txTo.Approved) {
+		t.Results.Modified = append(t.Results.Modified, TransactionModification{
+			TxID: txID,
+			From: txFrom,
+			To:   txTo,
+		})
+	}
+
+	return nil
+}
+
+func approvalsChanged(from, to []address.Address) bool {
+	if len(from) != len(to) {
+		return true
+	}
+	for idx := range from {
+		if from[idx] != to[idx] {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *transactionDiffer) Remove(key string, val *cbg.Deferred) error {
+	txID, err := abi.ParseIntKey(key)
+	if err != nil {
+		return err
+	}
+	tx, err := t.pre.decodeTransaction(val)
+	if err != nil {
+		return err
+	}
+	t.Results.Removed = append(t.Results.Removed, TransactionChange{
+		TxID: txID,
+		Tx:   tx,
+	})
+	return nil
+}

chain/actors/builtin/multisig/state.go

@@ -1,6 +1,7 @@
 package multisig
 
 import (
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -47,6 +48,10 @@ type State interface {
 	Signers() ([]address.Address, error)
 
 	ForEachPendingTxn(func(id int64, txn Transaction) error) error
+	PendingTxnChanged(State) (bool, error)
+
+	transactions() (adt.Map, error)
+	decodeTransaction(val *cbg.Deferred) (Transaction, error)
 }
 
 type Transaction = msig0.Transaction

chain/actors/builtin/multisig/state0.go

@@ -1,17 +1,20 @@
 package multisig
 
 import (
+	"bytes"
 	"encoding/binary"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 
 	msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig"
 	adt0 "github.com/filecoin-project/specs-actors/actors/util/adt"
+	multisig0 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
 )
 
 var _ State = (*state0)(nil)
@@ -68,3 +71,24 @@ func (s *state0) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err
 		return cb(txid, (Transaction)(out))
 	})
 }
+
+func (s *state0) PendingTxnChanged(other State) (bool, error) {
+	other0, ok := other.(*state0)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.PendingTxns.Equals(other0.PendingTxns), nil
+}
+
+func (s *state0) transactions() (adt.Map, error) {
+	return adt0.AsMap(s.store, s.PendingTxns)
+}
+
+func (s *state0) decodeTransaction(val *cbg.Deferred) (Transaction, error) {
+	var tx multisig0.Transaction
+	if err := tx.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return Transaction{}, err
+	}
+	return tx, nil
+}

chain/actors/builtin/multisig/state2.go

@@ -1,11 +1,13 @@
 package multisig
 
 import (
+	"bytes"
 	"encoding/binary"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/actors/adt"
@@ -68,3 +70,24 @@ func (s *state2) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err
 		return cb(txid, (Transaction)(out))
 	})
 }
+
+func (s *state2) PendingTxnChanged(other State) (bool, error) {
+	other2, ok := other.(*state2)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.PendingTxns.Equals(other2.PendingTxns), nil
+}
+
+func (s *state2) transactions() (adt.Map, error) {
+	return adt2.AsMap(s.store, s.PendingTxns)
+}
+
+func (s *state2) decodeTransaction(val *cbg.Deferred) (Transaction, error) {
+	var tx msig2.Transaction
+	if err := tx.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return Transaction{}, err
+	}
+	return tx, nil
+}

chain/actors/builtin/power/diff.go

@@ -0,0 +1,117 @@
+package power
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+)
+
+type ClaimChanges struct {
+	Added    []ClaimInfo
+	Modified []ClaimModification
+	Removed  []ClaimInfo
+}
+
+type ClaimModification struct {
+	Miner address.Address
+	From  Claim
+	To    Claim
+}
+
+type ClaimInfo struct {
+	Miner address.Address
+	Claim Claim
+}
+
+func DiffClaims(pre, cur State) (*ClaimChanges, error) {
+	results := new(ClaimChanges)
+
+	prec, err := pre.claims()
+	if err != nil {
+		return nil, err
+	}
+
+	curc, err := cur.claims()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := adt.DiffAdtMap(prec, curc, &claimDiffer{results, pre, cur}); err != nil {
+		return nil, err
+	}
+
+	return results, nil
+}
+
+type claimDiffer struct {
+	Results    *ClaimChanges
+	pre, after State
+}
+
+func (c *claimDiffer) AsKey(key string) (abi.Keyer, error) {
+	addr, err := address.NewFromBytes([]byte(key))
+	if err != nil {
+		return nil, err
+	}
+	return abi.AddrKey(addr), nil
+}
+
+func (c *claimDiffer) Add(key string, val *cbg.Deferred) error {
+	ci, err := c.after.decodeClaim(val)
+	if err != nil {
+		return err
+	}
+	addr, err := address.NewFromBytes([]byte(key))
+	if err != nil {
+		return err
+	}
+	c.Results.Added = append(c.Results.Added, ClaimInfo{
+		Miner: addr,
+		Claim: ci,
+	})
+	return nil
+}
+
+func (c *claimDiffer) Modify(key string, from, to *cbg.Deferred) error {
+	ciFrom, err := c.pre.decodeClaim(from)
+	if err != nil {
+		return err
+	}
+
+	ciTo, err := c.after.decodeClaim(to)
+	if err != nil {
+		return err
+	}
+
+	addr, err := address.NewFromBytes([]byte(key))
+	if err != nil {
+		return err
+	}
+
+	if ciFrom != ciTo {
+		c.Results.Modified = append(c.Results.Modified, ClaimModification{
+			Miner: addr,
+			From:  ciFrom,
+			To:    ciTo,
+		})
+	}
+	return nil
+}
+
+func (c *claimDiffer) Remove(key string, val *cbg.Deferred) error {
+	ci, err := c.after.decodeClaim(val)
+	if err != nil {
+		return err
+	}
+	addr, err := address.NewFromBytes([]byte(key))
+	if err != nil {
+		return err
+	}
+	c.Results.Removed = append(c.Results.Removed, ClaimInfo{
+		Miner: addr,
+		Claim: ci,
+	})
+	return nil
+}

chain/actors/builtin/power/power.go

@@ -4,6 +4,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
@@ -56,6 +57,11 @@ type State interface {
 	MinerNominalPowerMeetsConsensusMinimum(address.Address) (bool, error)
 	ListAllMiners() ([]address.Address, error)
 	ForEachClaim(func(miner address.Address, claim Claim) error) error
+	ClaimsChanged(State) (bool, error)
+
+	// Diff helpers. Used by Diff* functions internally.
+	claims() (adt.Map, error)
+	decodeClaim(*cbg.Deferred) (Claim, error)
 }
 
 type Claim struct {

chain/actors/builtin/power/v0.go

@@ -1,9 +1,12 @@
 package power
 
 import (
+	"bytes"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -115,3 +118,28 @@ func (s *state0) ForEachClaim(cb func(miner address.Address, claim Claim) error)
 		})
 	})
 }
+
+func (s *state0) ClaimsChanged(other State) (bool, error) {
+	other0, ok := other.(*state0)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Claims.Equals(other0.State.Claims), nil
+}
+
+func (s *state0) claims() (adt.Map, error) {
+	return adt0.AsMap(s.store, s.Claims)
+}
+
+func (s *state0) decodeClaim(val *cbg.Deferred) (Claim, error) {
+	var ci power0.Claim
+	if err := ci.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return Claim{}, err
+	}
+	return fromV0Claim(ci), nil
+}
+
+func fromV0Claim(v0 power0.Claim) Claim {
+	return (Claim)(v0)
+}

chain/actors/builtin/power/v2.go

@@ -1,9 +1,12 @@
 package power
 
 import (
+	"bytes"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -115,3 +118,31 @@ func (s *state2) ForEachClaim(cb func(miner address.Address, claim Claim) error)
 		})
 	})
 }
+
+func (s *state2) ClaimsChanged(other State) (bool, error) {
+	other2, ok := other.(*state2)
+	if !ok {
+		// treat an upgrade as a change, always
+		return true, nil
+	}
+	return !s.State.Claims.Equals(other2.State.Claims), nil
+}
+
+func (s *state2) claims() (adt.Map, error) {
+	return adt2.AsMap(s.store, s.Claims)
+}
+
+func (s *state2) decodeClaim(val *cbg.Deferred) (Claim, error) {
+	var ci power2.Claim
+	if err := ci.UnmarshalCBOR(bytes.NewReader(val.Raw)); err != nil {
+		return Claim{}, err
+	}
+	return fromV2Claim(ci), nil
+}
+
+func fromV2Claim(v2 power2.Claim) Claim {
+	return Claim{
+		RawBytePower:    v2.RawBytePower,
+		QualityAdjPower: v2.QualityAdjPower,
+	}
+}

chain/actors/builtin/reward/v2.go

@@ -28,7 +28,7 @@ type state2 struct {
 	store adt.Store
 }
 
-func (s *state2) ThisEpochReward() (abi.StoragePower, error) {
+func (s *state2) ThisEpochReward() (abi.TokenAmount, error) {
 	return s.State.ThisEpochReward, nil
 }
 
@@ -55,11 +55,11 @@ func (s *state2) EffectiveNetworkTime() (abi.ChainEpoch, error) {
 	return s.State.EffectiveNetworkTime, nil
 }
 
-func (s *state2) CumsumBaseline() (abi.StoragePower, error) {
+func (s *state2) CumsumBaseline() (reward2.Spacetime, error) {
 	return s.State.CumsumBaseline, nil
 }
 
-func (s *state2) CumsumRealized() (abi.StoragePower, error) {
+func (s *state2) CumsumRealized() (reward2.Spacetime, error) {
 	return s.State.CumsumRealized, nil
 }
 

chain/events/events_called.go

@@ -459,7 +459,7 @@ type messageEvents struct {
 	hcAPI headChangeAPI
 
 	lk       sync.RWMutex
-	matchers map[triggerID][]MsgMatchFunc
+	matchers map[triggerID]MsgMatchFunc
 }
 
 func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs eventAPI) messageEvents {
@@ -467,7 +467,7 @@ func newMessageEvents(ctx context.Context, hcAPI headChangeAPI, cs eventAPI) mes
 		ctx:      ctx,
 		cs:       cs,
 		hcAPI:    hcAPI,
-		matchers: map[triggerID][]MsgMatchFunc{},
+		matchers: make(map[triggerID]MsgMatchFunc),
 	}
 }
 
@@ -482,32 +482,23 @@ func (me *messageEvents) checkNewCalls(ts *types.TipSet) (map[triggerID]eventDat
 	me.lk.RLock()
 	defer me.lk.RUnlock()
 
+	// For each message in the tipset
 	res := make(map[triggerID]eventData)
 	me.messagesForTs(pts, func(msg *types.Message) {
 		// TODO: provide receipts
 
-		for tid, matchFns := range me.matchers {
-			var matched bool
-			var once bool
-			for _, matchFn := range matchFns {
-				matchOne, ok, err := matchFn(msg)
-				if err != nil {
-					log.Errorf("event matcher failed: %s", err)
-					continue
-				}
-				matched = ok
-				once = matchOne
-
-				if matched {
-					break
-				}
+		// Run each trigger's matcher against the message
+		for tid, matchFn := range me.matchers {
+			matched, err := matchFn(msg)
+			if err != nil {
+				log.Errorf("event matcher failed: %s", err)
+				continue
 			}
 
+			// If there was a match, include the message in the results for the
+			// trigger
 			if matched {
 				res[tid] = msg
-				if once {
-					break
-				}
 			}
 		}
 	})
@@ -555,7 +546,7 @@ func (me *messageEvents) messagesForTs(ts *types.TipSet, consume func(*types.Mes
 // `curH`-`ts.Height` = `confidence`
 type MsgHandler func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH abi.ChainEpoch) (more bool, err error)
 
-type MsgMatchFunc func(msg *types.Message) (matchOnce bool, matched bool, err error)
+type MsgMatchFunc func(msg *types.Message) (matched bool, err error)
 
 // Called registers a callback which is triggered when a specified method is
 //  called on an actor, or a timeout is reached.
@@ -607,7 +598,7 @@ func (me *messageEvents) Called(check CheckFunc, msgHnd MsgHandler, rev RevertHa
 
 	me.lk.Lock()
 	defer me.lk.Unlock()
-	me.matchers[id] = append(me.matchers[id], mf)
+	me.matchers[id] = mf
 
 	return nil
 }

chain/events/events_test.go

@@ -572,9 +572,9 @@ func TestAtChainedConfidenceNull(t *testing.T) {
 	require.Equal(t, false, reverted)
 }
 
-func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Message) (matchOnce bool, matched bool, err error) {
-	return func(msg *types.Message) (matchOnce bool, matched bool, err error) {
-		return true, to == msg.To && m == msg.Method, nil
+func matchAddrMethod(to address.Address, m abi.MethodNum) func(msg *types.Message) (matched bool, err error) {
+	return func(msg *types.Message) (matched bool, err error) {
+		return to == msg.To && m == msg.Method, nil
 	}
 }
 

chain/events/state/mock/api.go

@@ -0,0 +1,69 @@
+package test
+
+import (
+	"context"
+	"sync"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+)
+
+type MockAPI struct {
+	bs blockstore.Blockstore
+
+	lk                  sync.Mutex
+	ts                  map[types.TipSetKey]*types.Actor
+	stateGetActorCalled int
+}
+
+func NewMockAPI(bs blockstore.Blockstore) *MockAPI {
+	return &MockAPI{
+		bs: bs,
+		ts: make(map[types.TipSetKey]*types.Actor),
+	}
+}
+
+func (m *MockAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) {
+	return m.bs.Has(c)
+}
+
+func (m *MockAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
+	blk, err := m.bs.Get(c)
+	if err != nil {
+		return nil, xerrors.Errorf("blockstore get: %w", err)
+	}
+
+	return blk.RawData(), nil
+}
+
+func (m *MockAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
+	m.lk.Lock()
+	defer m.lk.Unlock()
+
+	m.stateGetActorCalled++
+	return m.ts[tsk], nil
+}
+
+func (m *MockAPI) StateGetActorCallCount() int {
+	m.lk.Lock()
+	defer m.lk.Unlock()
+
+	return m.stateGetActorCalled
+}
+
+func (m *MockAPI) ResetCallCounts() {
+	m.lk.Lock()
+	defer m.lk.Unlock()
+
+	m.stateGetActorCalled = 0
+}
+
+func (m *MockAPI) SetActor(tsk types.TipSetKey, act *types.Actor) {
+	m.lk.Lock()
+	defer m.lk.Unlock()
+
+	m.ts[tsk] = act
+}

chain/events/state/mock/state.go

@@ -0,0 +1,32 @@
+package test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
+	"github.com/filecoin-project/specs-actors/v2/actors/util/adt"
+	"github.com/stretchr/testify/require"
+)
+
+func CreateEmptyMarketState(t *testing.T, store adt.Store) *market.State {
+	emptyArrayCid, err := adt.MakeEmptyArray(store).Root()
+	require.NoError(t, err)
+	emptyMap, err := adt.MakeEmptyMap(store).Root()
+	require.NoError(t, err)
+	return market.ConstructState(emptyArrayCid, emptyMap, emptyMap)
+}
+
+func CreateDealAMT(ctx context.Context, t *testing.T, store adt.Store, deals map[abi.DealID]*market.DealState) cid.Cid {
+	root := adt.MakeEmptyArray(store)
+	for dealID, dealState := range deals {
+		err := root.Set(uint64(dealID), dealState)
+		require.NoError(t, err)
+	}
+	rootCid, err := root.Root()
+	require.NoError(t, err)
+	return rootCid
+}

chain/events/state/mock/tipset.go

@@ -0,0 +1,27 @@
+package test
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/ipfs/go-cid"
+)
+
+var dummyCid cid.Cid
+
+func init() {
+	dummyCid, _ = cid.Parse("bafkqaaa")
+}
+
+func MockTipset(minerAddr address.Address, timestamp uint64) (*types.TipSet, error) {
+	return types.NewTipSet([]*types.BlockHeader{{
+		Miner:                 minerAddr,
+		Height:                5,
+		ParentStateRoot:       dummyCid,
+		Messages:              dummyCid,
+		ParentMessageReceipts: dummyCid,
+		BlockSig:              &crypto.Signature{Type: crypto.SigTypeBLS},
+		BLSAggregate:          &crypto.Signature{Type: crypto.SigTypeBLS},
+		Timestamp:             timestamp,
+	}})
+}

chain/events/state/predicates_test.go

@@ -4,21 +4,19 @@ import (
 	"context"
 	"testing"
 
+	test "github.com/filecoin-project/lotus/chain/events/state/mock"
+
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 
 	"github.com/filecoin-project/go-bitfield"
 
-	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
-
 	"github.com/ipfs/go-cid"
 	cbornode "github.com/ipfs/go-ipld-cbor"
+	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/go-state-types/crypto"
-
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
 	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
@@ -36,39 +34,6 @@ func init() {
 	dummyCid, _ = cid.Parse("bafkqaaa")
 }
 
-type mockAPI struct {
-	ts map[types.TipSetKey]*types.Actor
-	bs bstore.Blockstore
-}
-
-func newMockAPI(bs bstore.Blockstore) *mockAPI {
-	return &mockAPI{
-		bs: bs,
-		ts: make(map[types.TipSetKey]*types.Actor),
-	}
-}
-
-func (m mockAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) {
-	return m.bs.Has(c)
-}
-
-func (m mockAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
-	blk, err := m.bs.Get(c)
-	if err != nil {
-		return nil, xerrors.Errorf("blockstore get: %w", err)
-	}
-
-	return blk.RawData(), nil
-}
-
-func (m mockAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
-	return m.ts[tsk], nil
-}
-
-func (m mockAPI) setActor(tsk types.TipSetKey, act *types.Actor) {
-	m.ts[tsk] = act
-}
-
 func TestMarketPredicates(t *testing.T) {
 	ctx := context.Background()
 	bs := bstore.NewTemporarySync()
@@ -177,14 +142,14 @@ func TestMarketPredicates(t *testing.T) {
 
 	minerAddr, err := address.NewFromString("t00")
 	require.NoError(t, err)
-	oldState, err := mockTipset(minerAddr, 1)
+	oldState, err := test.MockTipset(minerAddr, 1)
 	require.NoError(t, err)
-	newState, err := mockTipset(minerAddr, 2)
+	newState, err := test.MockTipset(minerAddr, 2)
 	require.NoError(t, err)
 
-	api := newMockAPI(bs)
-	api.setActor(oldState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: oldStateC})
-	api.setActor(newState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: newStateC})
+	api := test.NewMockAPI(bs)
+	api.SetActor(oldState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: oldStateC})
+	api.SetActor(newState.Key(), &types.Actor{Code: builtin2.StorageMarketActorCodeID, Head: newStateC})
 
 	t.Run("deal ID predicate", func(t *testing.T) {
 		preds := NewStatePredicates(api)
@@ -239,7 +204,7 @@ func TestMarketPredicates(t *testing.T) {
 			t.Fatal("No state change so this should not be called")
 			return false, nil, nil
 		})
-		marketState0 := createEmptyMarketState(t, store)
+		marketState0 := test.CreateEmptyMarketState(t, store)
 		marketCid, err := store.Put(ctx, marketState0)
 		require.NoError(t, err)
 		marketState, err := market.Load(store, &types.Actor{
@@ -352,7 +317,7 @@ func TestMarketPredicates(t *testing.T) {
 			t.Fatal("No state change so this should not be called")
 			return false, nil, nil
 		})
-		marketState0 := createEmptyMarketState(t, store)
+		marketState0 := test.CreateEmptyMarketState(t, store)
 		marketCid, err := store.Put(ctx, marketState0)
 		require.NoError(t, err)
 		marketState, err := market.Load(store, &types.Actor{
@@ -394,14 +359,14 @@ func TestMinerSectorChange(t *testing.T) {
 	newMinerC := createMinerState(ctx, t, store, owner, worker, []miner.SectorOnChainInfo{si1Ext, si2, si3})
 
 	minerAddr := nextIDAddrF()
-	oldState, err := mockTipset(minerAddr, 1)
+	oldState, err := test.MockTipset(minerAddr, 1)
 	require.NoError(t, err)
-	newState, err := mockTipset(minerAddr, 2)
+	newState, err := test.MockTipset(minerAddr, 2)
 	require.NoError(t, err)
 
-	api := newMockAPI(bs)
-	api.setActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: builtin2.StorageMinerActorCodeID})
-	api.setActor(newState.Key(), &types.Actor{Head: newMinerC, Code: builtin2.StorageMinerActorCodeID})
+	api := test.NewMockAPI(bs)
+	api.SetActor(oldState.Key(), &types.Actor{Head: oldMinerC, Code: builtin2.StorageMinerActorCodeID})
+	api.SetActor(newState.Key(), &types.Actor{Head: newMinerC, Code: builtin2.StorageMinerActorCodeID})
 
 	preds := NewStatePredicates(api)
 
@@ -449,29 +414,16 @@ func TestMinerSectorChange(t *testing.T) {
 	require.Equal(t, si1Ext, sectorChanges.Extended[0].From)
 }
 
-func mockTipset(minerAddr address.Address, timestamp uint64) (*types.TipSet, error) {
-	return types.NewTipSet([]*types.BlockHeader{{
-		Miner:                 minerAddr,
-		Height:                5,
-		ParentStateRoot:       dummyCid,
-		Messages:              dummyCid,
-		ParentMessageReceipts: dummyCid,
-		BlockSig:              &crypto.Signature{Type: crypto.SigTypeBLS},
-		BLSAggregate:          &crypto.Signature{Type: crypto.SigTypeBLS},
-		Timestamp:             timestamp,
-	}})
-}
-
 type balance struct {
 	available abi.TokenAmount
 	locked    abi.TokenAmount
 }
 
 func createMarketState(ctx context.Context, t *testing.T, store adt2.Store, deals map[abi.DealID]*market2.DealState, props map[abi.DealID]*market2.DealProposal, balances map[address.Address]balance) cid.Cid {
-	dealRootCid := createDealAMT(ctx, t, store, deals)
+	dealRootCid := test.CreateDealAMT(ctx, t, store, deals)
 	propRootCid := createProposalAMT(ctx, t, store, props)
 	balancesCids := createBalanceTable(ctx, t, store, balances)
-	state := createEmptyMarketState(t, store)
+	state := test.CreateEmptyMarketState(t, store)
 	state.States = dealRootCid
 	state.Proposals = propRootCid
 	state.EscrowTable = balancesCids[0]
@@ -482,25 +434,6 @@ func createMarketState(ctx context.Context, t *testing.T, store adt2.Store, deal
 	return stateC
 }
 
-func createEmptyMarketState(t *testing.T, store adt2.Store) *market2.State {
-	emptyArrayCid, err := adt2.MakeEmptyArray(store).Root()
-	require.NoError(t, err)
-	emptyMap, err := adt2.MakeEmptyMap(store).Root()
-	require.NoError(t, err)
-	return market2.ConstructState(emptyArrayCid, emptyMap, emptyMap)
-}
-
-func createDealAMT(ctx context.Context, t *testing.T, store adt2.Store, deals map[abi.DealID]*market2.DealState) cid.Cid {
-	root := adt2.MakeEmptyArray(store)
-	for dealID, dealState := range deals {
-		err := root.Set(uint64(dealID), dealState)
-		require.NoError(t, err)
-	}
-	rootCid, err := root.Root()
-	require.NoError(t, err)
-	return rootCid
-}
-
 func createProposalAMT(ctx context.Context, t *testing.T, store adt2.Store, props map[abi.DealID]*market2.DealProposal) cid.Cid {
 	root := adt2.MakeEmptyArray(store)
 	for dealID, prop := range props {

chain/events/utils.go

@@ -34,11 +34,11 @@ func (me *messageEvents) CheckMsg(ctx context.Context, smsg types.ChainMsg, hnd
 }
 
 func (me *messageEvents) MatchMsg(inmsg *types.Message) MsgMatchFunc {
-	return func(msg *types.Message) (matchOnce bool, matched bool, err error) {
+	return func(msg *types.Message) (matched bool, err error) {
 		if msg.From == inmsg.From && msg.Nonce == inmsg.Nonce && !inmsg.Equals(msg) {
-			return true, false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce)
+			return false, xerrors.Errorf("matching msg %s from %s, nonce %d: got duplicate origin/nonce msg %d", inmsg.Cid(), inmsg.From, inmsg.Nonce, msg.Nonce)
 		}
 
-		return true, inmsg.Equals(msg), nil
+		return inmsg.Equals(msg), nil
 	}
 }

chain/gen/gen.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"sync/atomic"
 	"time"
@@ -138,12 +139,20 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 		return nil, xerrors.Errorf("failed to get metadata datastore: %w", err)
 	}
 
-	bds, err := lr.Datastore("/chain")
+	bs, err := lr.Blockstore(repo.BlockstoreChain)
 	if err != nil {
-		return nil, xerrors.Errorf("failed to get blocks datastore: %w", err)
+		return nil, err
 	}
 
-	bs := mybs{blockstore.NewBlockstore(bds)}
+	defer func() {
+		if c, ok := bs.(io.Closer); ok {
+			if err := c.Close(); err != nil {
+				log.Warnf("failed to close blockstore: %s", err)
+			}
+		}
+	}()
+
+	bs = mybs{bs}
 
 	ks, err := lr.KeyStore()
 	if err != nil {
@@ -236,7 +245,7 @@ func NewGeneratorWithSectors(numSectors int) (*ChainGen, error) {
 		return nil, xerrors.Errorf("make genesis block failed: %w", err)
 	}
 
-	cs := store.NewChainStore(bs, ds, sys, j)
+	cs := store.NewChainStore(bs, bs, ds, sys, j)
 
 	genfb := &types.FullBlock{Header: genb.Genesis}
 	gents := store.NewFullTipSet([]*types.FullBlock{genfb})

chain/gen/genesis/genesis.go

@@ -482,7 +482,7 @@ func MakeGenesisBlock(ctx context.Context, j journal.Journal, bs bstore.Blocksto
 	}
 
 	// temp chainstore
-	cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys, j)
+	cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), sys, j)
 
 	// Verify PreSealed Data
 	stateroot, err = VerifyPreSealedData(ctx, cs, stateroot, template, keyIDs)

chain/market/cbor_gen.go

@@ -0,0 +1,112 @@
+// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
+
+package market
+
+import (
+	"fmt"
+	"io"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+)
+
+var _ = xerrors.Errorf
+
+var lengthBufFundedAddressState = []byte{131}
+
+func (t *FundedAddressState) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+	if _, err := w.Write(lengthBufFundedAddressState); err != nil {
+		return err
+	}
+
+	scratch := make([]byte, 9)
+
+	// t.Addr (address.Address) (struct)
+	if err := t.Addr.MarshalCBOR(w); err != nil {
+		return err
+	}
+
+	// t.AmtReserved (big.Int) (struct)
+	if err := t.AmtReserved.MarshalCBOR(w); err != nil {
+		return err
+	}
+
+	// t.MsgCid (cid.Cid) (struct)
+
+	if t.MsgCid == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.MsgCid); err != nil {
+			return xerrors.Errorf("failed to write cid field t.MsgCid: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error {
+	*t = FundedAddressState{}
+
+	br := cbg.GetPeeker(r)
+	scratch := make([]byte, 8)
+
+	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra != 3 {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	// t.Addr (address.Address) (struct)
+
+	{
+
+		if err := t.Addr.UnmarshalCBOR(br); err != nil {
+			return xerrors.Errorf("unmarshaling t.Addr: %w", err)
+		}
+
+	}
+	// t.AmtReserved (big.Int) (struct)
+
+	{
+
+		if err := t.AmtReserved.UnmarshalCBOR(br); err != nil {
+			return xerrors.Errorf("unmarshaling t.AmtReserved: %w", err)
+		}
+
+	}
+	// t.MsgCid (cid.Cid) (struct)
+
+	{
+
+		b, err := br.ReadByte()
+		if err != nil {
+			return err
+		}
+		if b != cbg.CborNull[0] {
+			if err := br.UnreadByte(); err != nil {
+				return err
+			}
+
+			c, err := cbg.ReadCid(br)
+			if err != nil {
+				return xerrors.Errorf("failed to read cid field t.MsgCid: %w", err)
+			}
+
+			t.MsgCid = &c
+		}
+
+	}
+	return nil
+}

chain/market/fundmanager.go

@@ -0,0 +1,707 @@
+package market
+
+import (
+	"context"
+	"sync"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl/full"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log"
+	"go.uber.org/fx"
+	"golang.org/x/xerrors"
+)
+
+var log = logging.Logger("market_adapter")
+
+// API is the fx dependencies need to run a fund manager
+type FundManagerAPI struct {
+	fx.In
+
+	full.StateAPI
+	full.MpoolAPI
+}
+
+// fundManagerAPI is the specific methods called by the FundManager
+// (used by the tests)
+type fundManagerAPI interface {
+	MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error)
+	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)
+	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)
+}
+
+// FundManager keeps track of funds in a set of addresses
+type FundManager struct {
+	ctx      context.Context
+	shutdown context.CancelFunc
+	api      fundManagerAPI
+	str      *Store
+
+	lk          sync.Mutex
+	fundedAddrs map[address.Address]*fundedAddress
+}
+
+func NewFundManager(lc fx.Lifecycle, api FundManagerAPI, ds dtypes.MetadataDS) *FundManager {
+	fm := newFundManager(&api, ds)
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			return fm.Start()
+		},
+		OnStop: func(ctx context.Context) error {
+			fm.Stop()
+			return nil
+		},
+	})
+	return fm
+}
+
+// newFundManager is used by the tests
+func newFundManager(api fundManagerAPI, ds datastore.Batching) *FundManager {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &FundManager{
+		ctx:         ctx,
+		shutdown:    cancel,
+		api:         api,
+		str:         newStore(ds),
+		fundedAddrs: make(map[address.Address]*fundedAddress),
+	}
+}
+
+func (fm *FundManager) Stop() {
+	fm.shutdown()
+}
+
+func (fm *FundManager) Start() error {
+	fm.lk.Lock()
+	defer fm.lk.Unlock()
+
+	// TODO:
+	// To save memory:
+	// - in State() only load addresses with in-progress messages
+	// - load the others just-in-time from getFundedAddress
+	// - delete(fm.fundedAddrs, addr) when the queue has been processed
+	return fm.str.forEach(func(state *FundedAddressState) {
+		fa := newFundedAddress(fm, state.Addr)
+		fa.state = state
+		fm.fundedAddrs[fa.state.Addr] = fa
+		fa.start()
+	})
+}
+
+// Creates a fundedAddress if it doesn't already exist, and returns it
+func (fm *FundManager) getFundedAddress(addr address.Address) *fundedAddress {
+	fm.lk.Lock()
+	defer fm.lk.Unlock()
+
+	fa, ok := fm.fundedAddrs[addr]
+	if !ok {
+		fa = newFundedAddress(fm, addr)
+		fm.fundedAddrs[addr] = fa
+	}
+	return fa
+}
+
+// Reserve adds amt to `reserved`. If there are not enough available funds for
+// the address, submits a message on chain to top up available funds.
+// Returns the cid of the message that was submitted on chain, or cid.Undef if
+// the required funds were already available.
+func (fm *FundManager) Reserve(ctx context.Context, wallet, addr address.Address, amt abi.TokenAmount) (cid.Cid, error) {
+	return fm.getFundedAddress(addr).reserve(ctx, wallet, amt)
+}
+
+// Subtract from `reserved`.
+func (fm *FundManager) Release(addr address.Address, amt abi.TokenAmount) error {
+	return fm.getFundedAddress(addr).release(amt)
+}
+
+// Withdraw unreserved funds. Only succeeds if there are enough unreserved
+// funds for the address.
+// Returns the cid of the message that was submitted on chain.
+func (fm *FundManager) Withdraw(ctx context.Context, wallet, addr address.Address, amt abi.TokenAmount) (cid.Cid, error) {
+	return fm.getFundedAddress(addr).withdraw(ctx, wallet, amt)
+}
+
+// FundedAddressState keeps track of the state of an address with funds in the
+// datastore
+type FundedAddressState struct {
+	Addr address.Address
+	// AmtReserved is the amount that must be kept in the address (cannot be
+	// withdrawn)
+	AmtReserved abi.TokenAmount
+	// MsgCid is the cid of an in-progress on-chain message
+	MsgCid *cid.Cid
+}
+
+// fundedAddress keeps track of the state and request queues for a
+// particular address
+type fundedAddress struct {
+	ctx context.Context
+	env *fundManagerEnvironment
+	str *Store
+
+	lk    sync.Mutex
+	state *FundedAddressState
+
+	// Note: These request queues are ephemeral, they are not saved to store
+	reservations []*fundRequest
+	releases     []*fundRequest
+	withdrawals  []*fundRequest
+
+	// Used by the tests
+	onProcessStartListener func() bool
+}
+
+func newFundedAddress(fm *FundManager, addr address.Address) *fundedAddress {
+	return &fundedAddress{
+		ctx: fm.ctx,
+		env: &fundManagerEnvironment{api: fm.api},
+		str: fm.str,
+		state: &FundedAddressState{
+			Addr:        addr,
+			AmtReserved: abi.NewTokenAmount(0),
+		},
+	}
+}
+
+// If there is an in-progress on-chain message, don't submit any more messages
+// on chain until it completes
+func (a *fundedAddress) start() {
+	a.lk.Lock()
+	defer a.lk.Unlock()
+
+	if a.state.MsgCid != nil {
+		a.debugf("restart: wait for %s", a.state.MsgCid)
+		a.startWaitForResults(*a.state.MsgCid)
+	}
+}
+
+func (a *fundedAddress) reserve(ctx context.Context, wallet address.Address, amt abi.TokenAmount) (cid.Cid, error) {
+	return a.requestAndWait(ctx, wallet, amt, &a.reservations)
+}
+
+func (a *fundedAddress) release(amt abi.TokenAmount) error {
+	_, err := a.requestAndWait(context.Background(), address.Undef, amt, &a.releases)
+	return err
+}
+
+func (a *fundedAddress) withdraw(ctx context.Context, wallet address.Address, amt abi.TokenAmount) (cid.Cid, error) {
+	return a.requestAndWait(ctx, wallet, amt, &a.withdrawals)
+}
+
+func (a *fundedAddress) requestAndWait(ctx context.Context, wallet address.Address, amt abi.TokenAmount, reqs *[]*fundRequest) (cid.Cid, error) {
+	// Create a request and add it to the request queue
+	req := newFundRequest(ctx, wallet, amt)
+
+	a.lk.Lock()
+	*reqs = append(*reqs, req)
+	a.lk.Unlock()
+
+	// Process the queue
+	go a.process()
+
+	// Wait for the results
+	select {
+	case <-ctx.Done():
+		return cid.Undef, ctx.Err()
+	case r := <-req.Result:
+		return r.msgCid, r.err
+	}
+}
+
+// Used by the tests
+func (a *fundedAddress) onProcessStart(fn func() bool) {
+	a.lk.Lock()
+	defer a.lk.Unlock()
+
+	a.onProcessStartListener = fn
+}
+
+// Process queued requests
+func (a *fundedAddress) process() {
+	a.lk.Lock()
+	defer a.lk.Unlock()
+
+	// Used by the tests
+	if a.onProcessStartListener != nil {
+		done := a.onProcessStartListener()
+		if !done {
+			return
+		}
+		a.onProcessStartListener = nil
+	}
+
+	// Check if we're still waiting for the response to a message
+	if a.state.MsgCid != nil {
+		return
+	}
+
+	// Check if there's anything to do
+	haveReservations := len(a.reservations) > 0 || len(a.releases) > 0
+	haveWithdrawals := len(a.withdrawals) > 0
+	if !haveReservations && !haveWithdrawals {
+		return
+	}
+
+	// Process reservations / releases
+	if haveReservations {
+		res, err := a.processReservations(a.reservations, a.releases)
+		if err == nil {
+			a.applyStateChange(res.msgCid, res.amtReserved)
+		}
+		a.reservations = filterOutProcessedReqs(a.reservations)
+		a.releases = filterOutProcessedReqs(a.releases)
+	}
+
+	// If there was no message sent on chain by adding reservations, and all
+	// reservations have completed processing, process withdrawals
+	if haveWithdrawals && a.state.MsgCid == nil && len(a.reservations) == 0 {
+		withdrawalCid, err := a.processWithdrawals(a.withdrawals)
+		if err == nil && withdrawalCid != cid.Undef {
+			a.applyStateChange(&withdrawalCid, types.EmptyInt)
+		}
+		a.withdrawals = filterOutProcessedReqs(a.withdrawals)
+	}
+
+	// If a message was sent on-chain
+	if a.state.MsgCid != nil {
+		// Start waiting for results of message (async)
+		a.startWaitForResults(*a.state.MsgCid)
+	}
+
+	// Process any remaining queued requests
+	go a.process()
+}
+
+// Filter out completed requests
+func filterOutProcessedReqs(reqs []*fundRequest) []*fundRequest {
+	filtered := make([]*fundRequest, 0, len(reqs))
+	for _, req := range reqs {
+		if !req.Completed() {
+			filtered = append(filtered, req)
+		}
+	}
+	return filtered
+}
+
+// Apply the results of processing queues and save to the datastore
+func (a *fundedAddress) applyStateChange(msgCid *cid.Cid, amtReserved abi.TokenAmount) {
+	a.state.MsgCid = msgCid
+	if !amtReserved.Nil() {
+		a.state.AmtReserved = amtReserved
+	}
+	a.saveState()
+}
+
+// Clear the pending message cid so that a new message can be sent
+func (a *fundedAddress) clearWaitState() {
+	a.state.MsgCid = nil
+	a.saveState()
+}
+
+// Save state to datastore
+func (a *fundedAddress) saveState() {
+	// Not much we can do if saving to the datastore fails, just log
+	err := a.str.save(a.state)
+	if err != nil {
+		log.Errorf("saving state to store for addr %s: %w", a.state.Addr, err)
+	}
+}
+
+// The result of processing the reservation / release queues
+type processResult struct {
+	// Requests that completed without adding funds
+	covered []*fundRequest
+	// Requests that added funds
+	added []*fundRequest
+
+	// The new reserved amount
+	amtReserved abi.TokenAmount
+	// The message cid, if a message was submitted on-chain
+	msgCid *cid.Cid
+}
+
+// process reservations and releases, and return the resulting changes to state
+func (a *fundedAddress) processReservations(reservations []*fundRequest, releases []*fundRequest) (pr *processResult, prerr error) {
+	// When the function returns
+	defer func() {
+		// If there's an error, mark all requests as errored
+		if prerr != nil {
+			for _, req := range append(reservations, releases...) {
+				req.Complete(cid.Undef, prerr)
+			}
+			return
+		}
+
+		// Complete all release requests
+		for _, req := range releases {
+			req.Complete(cid.Undef, nil)
+		}
+
+		// Complete all requests that were covered by released amounts
+		for _, req := range pr.covered {
+			req.Complete(cid.Undef, nil)
+		}
+
+		// If a message was sent
+		if pr.msgCid != nil {
+			// Complete all add funds requests
+			for _, req := range pr.added {
+				req.Complete(*pr.msgCid, nil)
+			}
+		}
+	}()
+
+	// Split reservations into those that are covered by released amounts,
+	// and those to add to the reserved amount.
+	// Note that we process requests from the same wallet in batches. So some
+	// requests may not be included in covered if they don't match the first
+	// covered request's wallet. These will be processed on a subsequent
+	// invocation of processReservations.
+	toCancel, toAdd, reservedDelta := splitReservations(reservations, releases)
+
+	// Apply the reserved delta to the reserved amount
+	reserved := types.BigAdd(a.state.AmtReserved, reservedDelta)
+	if reserved.LessThan(abi.NewTokenAmount(0)) {
+		reserved = abi.NewTokenAmount(0)
+	}
+	res := &processResult{
+		amtReserved: reserved,
+		covered:     toCancel,
+	}
+
+	// Work out the amount to add to the balance
+	amtToAdd := abi.NewTokenAmount(0)
+	if reserved.GreaterThan(abi.NewTokenAmount(0)) {
+		// Get available funds for address
+		avail, err := a.env.AvailableFunds(a.ctx, a.state.Addr)
+		if err != nil {
+			return res, err
+		}
+
+		// amount to add = new reserved amount - available
+		amtToAdd = types.BigSub(reserved, avail)
+		a.debugf("reserved %d - avail %d = to add %d", reserved, avail, amtToAdd)
+	}
+
+	// If there's nothing to add to the balance, bail out
+	if amtToAdd.LessThanEqual(abi.NewTokenAmount(0)) {
+		res.covered = append(res.covered, toAdd...)
+		return res, nil
+	}
+
+	// Add funds to address
+	a.debugf("add funds %d", amtToAdd)
+	addFundsCid, err := a.env.AddFunds(a.ctx, toAdd[0].Wallet, a.state.Addr, amtToAdd)
+	if err != nil {
+		return res, err
+	}
+
+	// Mark reservation requests as complete
+	res.added = toAdd
+
+	// Save the message CID to state
+	res.msgCid = &addFundsCid
+	return res, nil
+}
+
+// Split reservations into those that are under the total release amount
+// (covered) and those that exceed it (to add).
+// Note that we process requests from the same wallet in batches. So some
+// requests may not be included in covered if they don't match the first
+// covered request's wallet.
+func splitReservations(reservations []*fundRequest, releases []*fundRequest) ([]*fundRequest, []*fundRequest, abi.TokenAmount) {
+	toCancel := make([]*fundRequest, 0, len(reservations))
+	toAdd := make([]*fundRequest, 0, len(reservations))
+	toAddAmt := abi.NewTokenAmount(0)
+
+	// Sum release amounts
+	releaseAmt := abi.NewTokenAmount(0)
+	for _, req := range releases {
+		releaseAmt = types.BigAdd(releaseAmt, req.Amount())
+	}
+
+	// We only want to combine requests that come from the same wallet
+	batchWallet := address.Undef
+	for _, req := range reservations {
+		amt := req.Amount()
+
+		// If the amount to add to the reserve is cancelled out by a release
+		if amt.LessThanEqual(releaseAmt) {
+			// Cancel the request and update the release total
+			releaseAmt = types.BigSub(releaseAmt, amt)
+			toCancel = append(toCancel, req)
+			continue
+		}
+
+		// The amount to add is greater that the release total so we want
+		// to send an add funds request
+
+		// The first time the wallet will be undefined
+		if batchWallet == address.Undef {
+			batchWallet = req.Wallet
+		}
+		// If this request's wallet is the same as the batch wallet,
+		// the requests will be combined
+		if batchWallet == req.Wallet {
+			delta := types.BigSub(amt, releaseAmt)
+			toAddAmt = types.BigAdd(toAddAmt, delta)
+			releaseAmt = abi.NewTokenAmount(0)
+			toAdd = append(toAdd, req)
+		}
+	}
+
+	// The change in the reserved amount is "amount to add" - "amount to release"
+	reservedDelta := types.BigSub(toAddAmt, releaseAmt)
+
+	return toCancel, toAdd, reservedDelta
+}
+
+// process withdrawal queue
+func (a *fundedAddress) processWithdrawals(withdrawals []*fundRequest) (msgCid cid.Cid, prerr error) {
+	// If there's an error, mark all withdrawal requests as errored
+	defer func() {
+		if prerr != nil {
+			for _, req := range withdrawals {
+				req.Complete(cid.Undef, prerr)
+			}
+		}
+	}()
+
+	// Get the net available balance
+	avail, err := a.env.AvailableFunds(a.ctx, a.state.Addr)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	netAvail := types.BigSub(avail, a.state.AmtReserved)
+
+	// Fit as many withdrawals as possible into the available balance, and fail
+	// the rest
+	withdrawalAmt := abi.NewTokenAmount(0)
+	allowedAmt := abi.NewTokenAmount(0)
+	allowed := make([]*fundRequest, 0, len(a.withdrawals))
+	var batchWallet address.Address
+	for _, req := range a.withdrawals {
+		amt := req.Amount()
+		if amt.IsZero() {
+			// If the context for the request was cancelled, bail out
+			req.Complete(cid.Undef, err)
+			continue
+		}
+
+		// If the amount would exceed the available amount, complete the
+		// request with an error
+		newWithdrawalAmt := types.BigAdd(withdrawalAmt, amt)
+		if newWithdrawalAmt.GreaterThan(netAvail) {
+			err := xerrors.Errorf("insufficient funds for withdrawal of %d", amt)
+			a.debugf("%s", err)
+			req.Complete(cid.Undef, err)
+			continue
+		}
+
+		// If this is the first allowed withdrawal request in this batch, save
+		// its wallet address
+		if batchWallet == address.Undef {
+			batchWallet = req.Wallet
+		}
+		// If the request wallet doesn't match the batch wallet, bail out
+		// (the withdrawal will be processed after the current batch has
+		// completed)
+		if req.Wallet != batchWallet {
+			continue
+		}
+
+		// Include this withdrawal request in the batch
+		withdrawalAmt = newWithdrawalAmt
+		a.debugf("withdraw %d", amt)
+		allowed = append(allowed, req)
+		allowedAmt = types.BigAdd(allowedAmt, amt)
+	}
+
+	// Check if there is anything to withdraw.
+	// Note that if the context for a request is cancelled,
+	// req.Amount() returns zero
+	if allowedAmt.Equals(abi.NewTokenAmount(0)) {
+		// Mark allowed requests as complete
+		for _, req := range allowed {
+			req.Complete(cid.Undef, nil)
+		}
+		return cid.Undef, nil
+	}
+
+	// Withdraw funds
+	a.debugf("withdraw funds %d", allowedAmt)
+	withdrawFundsCid, err := a.env.WithdrawFunds(a.ctx, allowed[0].Wallet, a.state.Addr, allowedAmt)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	// Mark allowed requests as complete
+	for _, req := range allowed {
+		req.Complete(withdrawFundsCid, nil)
+	}
+
+	// Save the message CID to state
+	return withdrawFundsCid, nil
+}
+
+// asynchonously wait for results of message
+func (a *fundedAddress) startWaitForResults(msgCid cid.Cid) {
+	go func() {
+		err := a.env.WaitMsg(a.ctx, msgCid)
+		if err != nil {
+			// We don't really care about the results here, we're just waiting
+			// so as to only process one on-chain message at a time
+			log.Errorf("waiting for results of message %s for addr %s: %w", msgCid, a.state.Addr, err)
+		}
+
+		a.lk.Lock()
+		a.debugf("complete wait")
+		a.clearWaitState()
+		a.lk.Unlock()
+
+		a.process()
+	}()
+}
+
+func (a *fundedAddress) debugf(args ...interface{}) {
+	fmtStr := args[0].(string)
+	args = args[1:]
+	log.Debugf(a.state.Addr.String()+": "+fmtStr, args...)
+}
+
+// The result of a fund request
+type reqResult struct {
+	msgCid cid.Cid
+	err    error
+}
+
+// A request to change funds
+type fundRequest struct {
+	ctx       context.Context
+	amt       abi.TokenAmount
+	completed chan struct{}
+	Wallet    address.Address
+	Result    chan reqResult
+}
+
+func newFundRequest(ctx context.Context, wallet address.Address, amt abi.TokenAmount) *fundRequest {
+	return &fundRequest{
+		ctx:       ctx,
+		amt:       amt,
+		Wallet:    wallet,
+		Result:    make(chan reqResult),
+		completed: make(chan struct{}),
+	}
+}
+
+// Amount returns zero if the context has expired
+func (frp *fundRequest) Amount() abi.TokenAmount {
+	if frp.ctx.Err() != nil {
+		return abi.NewTokenAmount(0)
+	}
+	return frp.amt
+}
+
+// Complete is called with the message CID when the funds request has been
+// started or with the error if there was an error
+func (frp *fundRequest) Complete(msgCid cid.Cid, err error) {
+	select {
+	case <-frp.completed:
+	case <-frp.ctx.Done():
+	case frp.Result <- reqResult{msgCid: msgCid, err: err}:
+	}
+	close(frp.completed)
+}
+
+// Completed indicates if Complete has already been called
+func (frp *fundRequest) Completed() bool {
+	select {
+	case <-frp.completed:
+		return true
+	default:
+		return false
+	}
+}
+
+// fundManagerEnvironment simplifies some API calls
+type fundManagerEnvironment struct {
+	api fundManagerAPI
+}
+
+func (env *fundManagerEnvironment) AvailableFunds(ctx context.Context, addr address.Address) (abi.TokenAmount, error) {
+	bal, err := env.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
+	if err != nil {
+		return abi.NewTokenAmount(0), err
+	}
+
+	return types.BigSub(bal.Escrow, bal.Locked), nil
+}
+
+func (env *fundManagerEnvironment) AddFunds(
+	ctx context.Context,
+	wallet address.Address,
+	addr address.Address,
+	amt abi.TokenAmount,
+) (cid.Cid, error) {
+	params, err := actors.SerializeParams(&addr)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	smsg, aerr := env.api.MpoolPushMessage(ctx, &types.Message{
+		To:     market.Address,
+		From:   wallet,
+		Value:  amt,
+		Method: market.Methods.AddBalance,
+		Params: params,
+	}, nil)
+
+	if aerr != nil {
+		return cid.Undef, aerr
+	}
+
+	return smsg.Cid(), nil
+}
+
+func (env *fundManagerEnvironment) WithdrawFunds(
+	ctx context.Context,
+	wallet address.Address,
+	addr address.Address,
+	amt abi.TokenAmount,
+) (cid.Cid, error) {
+	params, err := actors.SerializeParams(&market.WithdrawBalanceParams{
+		ProviderOrClientAddress: addr,
+		Amount:                  amt,
+	})
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("serializing params: %w", err)
+	}
+
+	smsg, aerr := env.api.MpoolPushMessage(ctx, &types.Message{
+		To:     market.Address,
+		From:   wallet,
+		Value:  types.NewInt(0),
+		Method: market.Methods.WithdrawBalance,
+		Params: params,
+	}, nil)
+
+	if aerr != nil {
+		return cid.Undef, aerr
+	}
+
+	return smsg.Cid(), nil
+}
+
+func (env *fundManagerEnvironment) WaitMsg(ctx context.Context, c cid.Cid) error {
+	_, err := env.api.StateWaitMsg(ctx, c, build.MessageConfidence)
+	return err
+}

chain/market/fundmanager_test.go

@@ -0,0 +1,758 @@
+package market
+
+import (
+	"bytes"
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
+	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	ds_sync "github.com/ipfs/go-datastore/sync"
+	"github.com/stretchr/testify/require"
+)
+
+// TestFundManagerBasic verifies that the basic fund manager operations work
+func TestFundManagerBasic(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	// Reserve 10
+	// balance:  0 -> 10
+	// reserved: 0 -> 10
+	amt := abi.NewTokenAmount(10)
+	sentinel, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	msg := s.mockApi.getSentMessage(sentinel)
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, amt)
+
+	s.mockApi.completeMsg(sentinel)
+
+	// Reserve 7
+	// balance:  10 -> 17
+	// reserved: 10 -> 17
+	amt = abi.NewTokenAmount(7)
+	sentinel, err = s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	msg = s.mockApi.getSentMessage(sentinel)
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, amt)
+
+	s.mockApi.completeMsg(sentinel)
+
+	// Release 5
+	// balance:  17
+	// reserved: 17 -> 12
+	amt = abi.NewTokenAmount(5)
+	err = s.fm.Release(s.acctAddr, amt)
+	require.NoError(t, err)
+
+	// Withdraw 2
+	// balance:  17 -> 15
+	// reserved: 12
+	amt = abi.NewTokenAmount(2)
+	sentinel, err = s.fm.Withdraw(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	msg = s.mockApi.getSentMessage(sentinel)
+	checkWithdrawMessageFields(t, msg, s.walletAddr, s.acctAddr, amt)
+
+	s.mockApi.completeMsg(sentinel)
+
+	// Reserve 3
+	// balance:  15
+	// reserved: 12 -> 15
+	// Note: reserved (15) is <= balance (15) so should not send on-chain
+	// message
+	msgCount := s.mockApi.messageCount()
+	amt = abi.NewTokenAmount(3)
+	sentinel, err = s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+	require.Equal(t, msgCount, s.mockApi.messageCount())
+	require.Equal(t, sentinel, cid.Undef)
+
+	// Reserve 1
+	// balance:  15 -> 16
+	// reserved: 15 -> 16
+	// Note: reserved (16) is above balance (15) so *should* send on-chain
+	// message to top up balance
+	amt = abi.NewTokenAmount(1)
+	topUp := abi.NewTokenAmount(1)
+	sentinel, err = s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	s.mockApi.completeMsg(sentinel)
+	msg = s.mockApi.getSentMessage(sentinel)
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, topUp)
+
+	// Withdraw 1
+	// balance:  16
+	// reserved: 16
+	// Note: Expect failure because there is no available balance to withdraw:
+	// balance - reserved = 16 - 16 = 0
+	amt = abi.NewTokenAmount(1)
+	sentinel, err = s.fm.Withdraw(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.Error(t, err)
+}
+
+// TestFundManagerParallel verifies that operations can be run in parallel
+func TestFundManagerParallel(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	// Reserve 10
+	amt := abi.NewTokenAmount(10)
+	sentinelReserve10, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	// Wait until all the subsequent requests are queued up
+	queueReady := make(chan struct{})
+	fa := s.fm.getFundedAddress(s.acctAddr)
+	fa.onProcessStart(func() bool {
+		if len(fa.withdrawals) == 1 && len(fa.reservations) == 2 && len(fa.releases) == 1 {
+			close(queueReady)
+			return true
+		}
+		return false
+	})
+
+	// Withdraw 5 (should not run until after reserves / releases)
+	withdrawReady := make(chan error)
+	go func() {
+		amt = abi.NewTokenAmount(5)
+		_, err := s.fm.Withdraw(s.ctx, s.walletAddr, s.acctAddr, amt)
+		withdrawReady <- err
+	}()
+
+	reserveSentinels := make(chan cid.Cid)
+
+	// Reserve 3
+	go func() {
+		amt := abi.NewTokenAmount(3)
+		sentinelReserve3, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+		require.NoError(t, err)
+		reserveSentinels <- sentinelReserve3
+	}()
+
+	// Reserve 5
+	go func() {
+		amt := abi.NewTokenAmount(5)
+		sentinelReserve5, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+		require.NoError(t, err)
+		reserveSentinels <- sentinelReserve5
+	}()
+
+	// Release 2
+	go func() {
+		amt := abi.NewTokenAmount(2)
+		err = s.fm.Release(s.acctAddr, amt)
+		require.NoError(t, err)
+	}()
+
+	// Everything is queued up
+	<-queueReady
+
+	// Complete the "Reserve 10" message
+	s.mockApi.completeMsg(sentinelReserve10)
+	msg := s.mockApi.getSentMessage(sentinelReserve10)
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, abi.NewTokenAmount(10))
+
+	// The other requests should now be combined and be submitted on-chain as
+	// a single message
+	rs1 := <-reserveSentinels
+	rs2 := <-reserveSentinels
+	require.Equal(t, rs1, rs2)
+
+	// Withdraw should not have been called yet, because reserve / release
+	// requests run first
+	select {
+	case <-withdrawReady:
+		require.Fail(t, "Withdraw should run after reserve / release")
+	default:
+	}
+
+	// Complete the message
+	s.mockApi.completeMsg(rs1)
+	msg = s.mockApi.getSentMessage(rs1)
+
+	// "Reserve 3" +3
+	// "Reserve 5" +5
+	// "Release 2" -2
+	// Result:      6
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, abi.NewTokenAmount(6))
+
+	// Expect withdraw to fail because not enough available funds
+	err = <-withdrawReady
+	require.Error(t, err)
+}
+
+// TestFundManagerReserveByWallet verifies that reserve requests are grouped by wallet
+func TestFundManagerReserveByWallet(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	walletAddrA, err := s.wllt.WalletNew(context.Background(), types.KTSecp256k1)
+	require.NoError(t, err)
+	walletAddrB, err := s.wllt.WalletNew(context.Background(), types.KTSecp256k1)
+	require.NoError(t, err)
+
+	// Wait until all the reservation requests are queued up
+	walletAQueuedUp := make(chan struct{})
+	queueReady := make(chan struct{})
+	fa := s.fm.getFundedAddress(s.acctAddr)
+	fa.onProcessStart(func() bool {
+		if len(fa.reservations) == 1 {
+			close(walletAQueuedUp)
+		}
+		if len(fa.reservations) == 3 {
+			close(queueReady)
+			return true
+		}
+		return false
+	})
+
+	type reserveResult struct {
+		ws  cid.Cid
+		err error
+	}
+	results := make(chan *reserveResult)
+
+	amtA1 := abi.NewTokenAmount(1)
+	go func() {
+		// Wallet A: Reserve 1
+		sentinelA1, err := s.fm.Reserve(s.ctx, walletAddrA, s.acctAddr, amtA1)
+		results <- &reserveResult{
+			ws:  sentinelA1,
+			err: err,
+		}
+	}()
+
+	amtB1 := abi.NewTokenAmount(2)
+	amtB2 := abi.NewTokenAmount(3)
+	go func() {
+		// Wait for reservation for wallet A to be queued up
+		<-walletAQueuedUp
+
+		// Wallet B: Reserve 2
+		go func() {
+			sentinelB1, err := s.fm.Reserve(s.ctx, walletAddrB, s.acctAddr, amtB1)
+			results <- &reserveResult{
+				ws:  sentinelB1,
+				err: err,
+			}
+		}()
+
+		// Wallet B: Reserve 3
+		sentinelB2, err := s.fm.Reserve(s.ctx, walletAddrB, s.acctAddr, amtB2)
+		results <- &reserveResult{
+			ws:  sentinelB2,
+			err: err,
+		}
+	}()
+
+	// All reservation requests are queued up
+	<-queueReady
+
+	resA := <-results
+	sentinelA1 := resA.ws
+
+	// Should send to wallet A
+	msg := s.mockApi.getSentMessage(sentinelA1)
+	checkAddMessageFields(t, msg, walletAddrA, s.acctAddr, amtA1)
+
+	// Complete wallet A message
+	s.mockApi.completeMsg(sentinelA1)
+
+	resB1 := <-results
+	resB2 := <-results
+	require.NoError(t, resB1.err)
+	require.NoError(t, resB2.err)
+	sentinelB1 := resB1.ws
+	sentinelB2 := resB2.ws
+
+	// Should send different message to wallet B
+	require.NotEqual(t, sentinelA1, sentinelB1)
+	// Should be single message combining amount 1 and 2
+	require.Equal(t, sentinelB1, sentinelB2)
+	msg = s.mockApi.getSentMessage(sentinelB1)
+	checkAddMessageFields(t, msg, walletAddrB, s.acctAddr, types.BigAdd(amtB1, amtB2))
+}
+
+// TestFundManagerWithdrawal verifies that as many withdraw operations as
+// possible are processed
+func TestFundManagerWithdrawalLimit(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	// Reserve 10
+	amt := abi.NewTokenAmount(10)
+	sentinelReserve10, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	// Complete the "Reserve 10" message
+	s.mockApi.completeMsg(sentinelReserve10)
+
+	// Release 10
+	err = s.fm.Release(s.acctAddr, amt)
+	require.NoError(t, err)
+
+	// Queue up withdraw requests
+	queueReady := make(chan struct{})
+	fa := s.fm.getFundedAddress(s.acctAddr)
+	withdrawalReqTotal := 3
+	withdrawalReqEnqueued := 0
+	withdrawalReqQueue := make(chan func(), withdrawalReqTotal)
+	fa.onProcessStart(func() bool {
+		// If a new withdrawal request was enqueued
+		if len(fa.withdrawals) > withdrawalReqEnqueued {
+			withdrawalReqEnqueued++
+
+			// Pop the next request and run it
+			select {
+			case fn := <-withdrawalReqQueue:
+				go fn()
+			default:
+			}
+		}
+		// Once all the requests have arrived, we're ready to process the queue
+		if withdrawalReqEnqueued == withdrawalReqTotal {
+			close(queueReady)
+			return true
+		}
+		return false
+	})
+
+	type withdrawResult struct {
+		reqIndex int
+		ws       cid.Cid
+		err      error
+	}
+	withdrawRes := make(chan *withdrawResult)
+
+	// Queue up three "Withdraw 5" requests
+	enqueuedCount := 0
+	for i := 0; i < withdrawalReqTotal; i++ {
+		withdrawalReqQueue <- func() {
+			idx := enqueuedCount
+			enqueuedCount++
+
+			amt := abi.NewTokenAmount(5)
+			ws, err := s.fm.Withdraw(s.ctx, s.walletAddr, s.acctAddr, amt)
+			withdrawRes <- &withdrawResult{reqIndex: idx, ws: ws, err: err}
+		}
+	}
+	// Start the first request
+	fn := <-withdrawalReqQueue
+	go fn()
+
+	// All withdrawal requests are queued up and ready to be processed
+	<-queueReady
+
+	// Organize results in request order
+	results := make([]*withdrawResult, withdrawalReqTotal)
+	for i := 0; i < 3; i++ {
+		res := <-withdrawRes
+		results[res.reqIndex] = res
+	}
+
+	// Available 10
+	// Withdraw 5
+	// Expect Success
+	require.NoError(t, results[0].err)
+	// Available 5
+	// Withdraw 5
+	// Expect Success
+	require.NoError(t, results[1].err)
+	// Available 0
+	// Withdraw 5
+	// Expect FAIL
+	require.Error(t, results[2].err)
+
+	// Expect withdrawal requests that fit under reserved amount to be combined
+	// into a single message on-chain
+	require.Equal(t, results[0].ws, results[1].ws)
+}
+
+// TestFundManagerWithdrawByWallet verifies that withdraw requests are grouped by wallet
+func TestFundManagerWithdrawByWallet(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	walletAddrA, err := s.wllt.WalletNew(context.Background(), types.KTSecp256k1)
+	require.NoError(t, err)
+	walletAddrB, err := s.wllt.WalletNew(context.Background(), types.KTSecp256k1)
+	require.NoError(t, err)
+
+	// Reserve 10
+	reserveAmt := abi.NewTokenAmount(10)
+	sentinelReserve, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, reserveAmt)
+	require.NoError(t, err)
+	s.mockApi.completeMsg(sentinelReserve)
+
+	time.Sleep(10 * time.Millisecond)
+
+	// Release 10
+	err = s.fm.Release(s.acctAddr, reserveAmt)
+	require.NoError(t, err)
+
+	type withdrawResult struct {
+		ws  cid.Cid
+		err error
+	}
+	results := make(chan *withdrawResult)
+
+	// Wait until withdrawals are queued up
+	walletAQueuedUp := make(chan struct{})
+	queueReady := make(chan struct{})
+	withdrawalCount := 0
+	fa := s.fm.getFundedAddress(s.acctAddr)
+	fa.onProcessStart(func() bool {
+		if len(fa.withdrawals) == withdrawalCount {
+			return false
+		}
+		withdrawalCount = len(fa.withdrawals)
+
+		if withdrawalCount == 1 {
+			close(walletAQueuedUp)
+		} else if withdrawalCount == 3 {
+			close(queueReady)
+			return true
+		}
+		return false
+	})
+
+	amtA1 := abi.NewTokenAmount(1)
+	go func() {
+		// Wallet A: Withdraw 1
+		sentinelA1, err := s.fm.Withdraw(s.ctx, walletAddrA, s.acctAddr, amtA1)
+		results <- &withdrawResult{
+			ws:  sentinelA1,
+			err: err,
+		}
+	}()
+
+	amtB1 := abi.NewTokenAmount(2)
+	amtB2 := abi.NewTokenAmount(3)
+	go func() {
+		// Wait until withdraw for wallet A is queued up
+		<-walletAQueuedUp
+
+		// Wallet B: Withdraw 2
+		go func() {
+			sentinelB1, err := s.fm.Withdraw(s.ctx, walletAddrB, s.acctAddr, amtB1)
+			results <- &withdrawResult{
+				ws:  sentinelB1,
+				err: err,
+			}
+		}()
+
+		// Wallet B: Withdraw 3
+		sentinelB2, err := s.fm.Withdraw(s.ctx, walletAddrB, s.acctAddr, amtB2)
+		results <- &withdrawResult{
+			ws:  sentinelB2,
+			err: err,
+		}
+	}()
+
+	// Withdrawals are queued up
+	<-queueReady
+
+	// Should withdraw from wallet A first
+	resA1 := <-results
+	sentinelA1 := resA1.ws
+	msg := s.mockApi.getSentMessage(sentinelA1)
+	checkWithdrawMessageFields(t, msg, walletAddrA, s.acctAddr, amtA1)
+
+	// Complete wallet A message
+	s.mockApi.completeMsg(sentinelA1)
+
+	resB1 := <-results
+	resB2 := <-results
+	require.NoError(t, resB1.err)
+	require.NoError(t, resB2.err)
+	sentinelB1 := resB1.ws
+	sentinelB2 := resB2.ws
+
+	// Should send different message for wallet B from wallet A
+	require.NotEqual(t, sentinelA1, sentinelB1)
+	// Should be single message combining amount 1 and 2
+	require.Equal(t, sentinelB1, sentinelB2)
+	msg = s.mockApi.getSentMessage(sentinelB1)
+	checkWithdrawMessageFields(t, msg, walletAddrB, s.acctAddr, types.BigAdd(amtB1, amtB2))
+}
+
+// TestFundManagerRestart verifies that waiting for incomplete requests resumes
+// on restart
+func TestFundManagerRestart(t *testing.T) {
+	s := setup(t)
+	defer s.fm.Stop()
+
+	acctAddr2 := tutils.NewActorAddr(t, "addr2")
+
+	// Address 1: Reserve 10
+	amt := abi.NewTokenAmount(10)
+	sentinelAddr1, err := s.fm.Reserve(s.ctx, s.walletAddr, s.acctAddr, amt)
+	require.NoError(t, err)
+
+	msg := s.mockApi.getSentMessage(sentinelAddr1)
+	checkAddMessageFields(t, msg, s.walletAddr, s.acctAddr, amt)
+
+	// Address 2: Reserve 7
+	amt2 := abi.NewTokenAmount(7)
+	sentinelAddr2Res7, err := s.fm.Reserve(s.ctx, s.walletAddr, acctAddr2, amt2)
+	require.NoError(t, err)
+
+	msg2 := s.mockApi.getSentMessage(sentinelAddr2Res7)
+	checkAddMessageFields(t, msg2, s.walletAddr, acctAddr2, amt2)
+
+	// Complete "Address 1: Reserve 10"
+	s.mockApi.completeMsg(sentinelAddr1)
+
+	// Give the completed state a moment to be stored before restart
+	time.Sleep(time.Millisecond * 10)
+
+	// Restart
+	mockApiAfter := s.mockApi
+	fmAfter := newFundManager(mockApiAfter, s.ds)
+	err = fmAfter.Start()
+	require.NoError(t, err)
+
+	amt3 := abi.NewTokenAmount(9)
+	reserveSentinel := make(chan cid.Cid)
+	go func() {
+		// Address 2: Reserve 9
+		sentinel3, err := fmAfter.Reserve(s.ctx, s.walletAddr, acctAddr2, amt3)
+		require.NoError(t, err)
+		reserveSentinel <- sentinel3
+	}()
+
+	// Expect no message to be sent, because still waiting for previous
+	// message "Address 2: Reserve 7" to complete on-chain
+	select {
+	case <-reserveSentinel:
+		require.Fail(t, "Expected no message to be sent")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Complete "Address 2: Reserve 7"
+	mockApiAfter.completeMsg(sentinelAddr2Res7)
+
+	// Expect waiting message to now be sent
+	sentinel3 := <-reserveSentinel
+	msg3 := mockApiAfter.getSentMessage(sentinel3)
+	checkAddMessageFields(t, msg3, s.walletAddr, acctAddr2, amt3)
+}
+
+type scaffold struct {
+	ctx        context.Context
+	ds         *ds_sync.MutexDatastore
+	wllt       *wallet.LocalWallet
+	walletAddr address.Address
+	acctAddr   address.Address
+	mockApi    *mockFundManagerAPI
+	fm         *FundManager
+}
+
+func setup(t *testing.T) *scaffold {
+	ctx := context.Background()
+
+	wllt, err := wallet.NewWallet(wallet.NewMemKeyStore())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	walletAddr, err := wllt.WalletNew(context.Background(), types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	acctAddr := tutils.NewActorAddr(t, "addr")
+
+	mockApi := newMockFundManagerAPI(walletAddr)
+	dstore := ds_sync.MutexWrap(ds.NewMapDatastore())
+	fm := newFundManager(mockApi, dstore)
+	return &scaffold{
+		ctx:        ctx,
+		ds:         dstore,
+		wllt:       wllt,
+		walletAddr: walletAddr,
+		acctAddr:   acctAddr,
+		mockApi:    mockApi,
+		fm:         fm,
+	}
+}
+
+func checkAddMessageFields(t *testing.T, msg *types.Message, from address.Address, to address.Address, amt abi.TokenAmount) {
+	require.Equal(t, from, msg.From)
+	require.Equal(t, market.Address, msg.To)
+	require.Equal(t, amt, msg.Value)
+
+	var paramsTo address.Address
+	err := paramsTo.UnmarshalCBOR(bytes.NewReader(msg.Params))
+	require.NoError(t, err)
+	require.Equal(t, to, paramsTo)
+}
+
+func checkWithdrawMessageFields(t *testing.T, msg *types.Message, from address.Address, addr address.Address, amt abi.TokenAmount) {
+	require.Equal(t, from, msg.From)
+	require.Equal(t, market.Address, msg.To)
+	require.Equal(t, abi.NewTokenAmount(0), msg.Value)
+
+	var params market.WithdrawBalanceParams
+	err := params.UnmarshalCBOR(bytes.NewReader(msg.Params))
+	require.NoError(t, err)
+	require.Equal(t, addr, params.ProviderOrClientAddress)
+	require.Equal(t, amt, params.Amount)
+}
+
+type sentMsg struct {
+	msg   *types.SignedMessage
+	ready chan struct{}
+}
+
+type mockFundManagerAPI struct {
+	wallet address.Address
+
+	lk            sync.Mutex
+	escrow        map[address.Address]abi.TokenAmount
+	sentMsgs      map[cid.Cid]*sentMsg
+	completedMsgs map[cid.Cid]struct{}
+	waitingFor    map[cid.Cid]chan struct{}
+}
+
+func newMockFundManagerAPI(wallet address.Address) *mockFundManagerAPI {
+	return &mockFundManagerAPI{
+		wallet:        wallet,
+		escrow:        make(map[address.Address]abi.TokenAmount),
+		sentMsgs:      make(map[cid.Cid]*sentMsg),
+		completedMsgs: make(map[cid.Cid]struct{}),
+		waitingFor:    make(map[cid.Cid]chan struct{}),
+	}
+}
+
+func (mapi *mockFundManagerAPI) MpoolPushMessage(ctx context.Context, message *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
+	mapi.lk.Lock()
+	defer mapi.lk.Unlock()
+
+	smsg := &types.SignedMessage{Message: *message}
+	mapi.sentMsgs[smsg.Cid()] = &sentMsg{msg: smsg, ready: make(chan struct{})}
+
+	return smsg, nil
+}
+
+func (mapi *mockFundManagerAPI) getSentMessage(c cid.Cid) *types.Message {
+	mapi.lk.Lock()
+	defer mapi.lk.Unlock()
+
+	for i := 0; i < 1000; i++ {
+		if pending, ok := mapi.sentMsgs[c]; ok {
+			return &pending.msg.Message
+		}
+		time.Sleep(time.Millisecond)
+	}
+	panic("expected message to be sent")
+}
+
+func (mapi *mockFundManagerAPI) messageCount() int {
+	mapi.lk.Lock()
+	defer mapi.lk.Unlock()
+
+	return len(mapi.sentMsgs)
+}
+
+func (mapi *mockFundManagerAPI) completeMsg(msgCid cid.Cid) {
+	mapi.lk.Lock()
+
+	pmsg, ok := mapi.sentMsgs[msgCid]
+	if ok {
+		if pmsg.msg.Message.Method == market.Methods.AddBalance {
+			var escrowAcct address.Address
+			err := escrowAcct.UnmarshalCBOR(bytes.NewReader(pmsg.msg.Message.Params))
+			if err != nil {
+				panic(err)
+			}
+
+			escrow := mapi.getEscrow(escrowAcct)
+			before := escrow
+			escrow = types.BigAdd(escrow, pmsg.msg.Message.Value)
+			mapi.escrow[escrowAcct] = escrow
+			log.Debugf("%s:   escrow %d -> %d", escrowAcct, before, escrow)
+		} else {
+			var params market.WithdrawBalanceParams
+			err := params.UnmarshalCBOR(bytes.NewReader(pmsg.msg.Message.Params))
+			if err != nil {
+				panic(err)
+			}
+			escrowAcct := params.ProviderOrClientAddress
+
+			escrow := mapi.getEscrow(escrowAcct)
+			before := escrow
+			escrow = types.BigSub(escrow, params.Amount)
+			mapi.escrow[escrowAcct] = escrow
+			log.Debugf("%s:   escrow %d -> %d", escrowAcct, before, escrow)
+		}
+	}
+
+	mapi.completedMsgs[msgCid] = struct{}{}
+
+	ready, ok := mapi.waitingFor[msgCid]
+
+	mapi.lk.Unlock()
+
+	if ok {
+		close(ready)
+	}
+}
+
+func (mapi *mockFundManagerAPI) StateMarketBalance(ctx context.Context, a address.Address, key types.TipSetKey) (api.MarketBalance, error) {
+	mapi.lk.Lock()
+	defer mapi.lk.Unlock()
+
+	return api.MarketBalance{
+		Locked: abi.NewTokenAmount(0),
+		Escrow: mapi.getEscrow(a),
+	}, nil
+}
+
+func (mapi *mockFundManagerAPI) getEscrow(a address.Address) abi.TokenAmount {
+	escrow := mapi.escrow[a]
+	if escrow.Nil() {
+		return abi.NewTokenAmount(0)
+	}
+	return escrow
+}
+
+func (mapi *mockFundManagerAPI) StateWaitMsg(ctx context.Context, c cid.Cid, confidence uint64) (*api.MsgLookup, error) {
+	res := &api.MsgLookup{
+		Message: c,
+		Receipt: types.MessageReceipt{
+			ExitCode: 0,
+			Return:   nil,
+		},
+	}
+	ready := make(chan struct{})
+
+	mapi.lk.Lock()
+	_, ok := mapi.completedMsgs[c]
+	if !ok {
+		mapi.waitingFor[c] = ready
+	}
+	mapi.lk.Unlock()
+
+	if !ok {
+		select {
+		case <-ctx.Done():
+		case <-ready:
+		}
+	}
+	return res, nil
+}

chain/market/fundmgr.go

@@ -1,166 +0,0 @@
-package market
-
-import (
-	"context"
-	"sync"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
-	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
-	"go.uber.org/fx"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/events/state"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/impl/full"
-)
-
-var log = logging.Logger("market_adapter")
-
-// API is the dependencies need to run a fund manager
-type API struct {
-	fx.In
-
-	full.ChainAPI
-	full.StateAPI
-	full.MpoolAPI
-}
-
-// FundMgr monitors available balances and adds funds when EnsureAvailable is called
-type FundMgr struct {
-	api fundMgrAPI
-
-	lk        sync.RWMutex
-	available map[address.Address]types.BigInt
-}
-
-// StartFundManager creates a new fund manager and sets up event hooks to manage state changes
-func StartFundManager(lc fx.Lifecycle, api API) *FundMgr {
-	fm := newFundMgr(&api)
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			ev := events.NewEvents(ctx, &api)
-			preds := state.NewStatePredicates(&api)
-			dealDiffFn := preds.OnStorageMarketActorChanged(preds.OnBalanceChanged(preds.AvailableBalanceChangedForAddresses(fm.getAddresses)))
-			match := func(oldTs, newTs *types.TipSet) (bool, events.StateChange, error) {
-				return dealDiffFn(ctx, oldTs.Key(), newTs.Key())
-			}
-			return ev.StateChanged(fm.checkFunc, fm.stateChanged, fm.revert, 0, events.NoTimeout, match)
-		},
-	})
-	return fm
-}
-
-type fundMgrAPI interface {
-	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)
-	MpoolPushMessage(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error)
-	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error)
-}
-
-func newFundMgr(api fundMgrAPI) *FundMgr {
-	return &FundMgr{
-		api:       api,
-		available: map[address.Address]types.BigInt{},
-	}
-}
-
-// checkFunc tells the events api to simply proceed (we always want to watch)
-func (fm *FundMgr) checkFunc(ts *types.TipSet) (done bool, more bool, err error) {
-	return false, true, nil
-}
-
-// revert handles reverts to balances
-func (fm *FundMgr) revert(ctx context.Context, ts *types.TipSet) error {
-	// TODO: Is it ok to just ignore this?
-	log.Warn("balance change reverted; TODO: actually handle this!")
-	return nil
-}
-
-// stateChanged handles balance changes monitored on the chain from one tipset to the next
-func (fm *FundMgr) stateChanged(ts *types.TipSet, ts2 *types.TipSet, states events.StateChange, h abi.ChainEpoch) (more bool, err error) {
-	changedBalances, ok := states.(state.ChangedBalances)
-	if !ok {
-		panic("Expected state.ChangedBalances")
-	}
-	// overwrite our in memory cache with new values from chain (chain is canonical)
-	fm.lk.Lock()
-	for addr, balanceChange := range changedBalances {
-		if fm.available[addr].Int != nil {
-			log.Infof("State balance change recorded, prev: %s, new: %s", fm.available[addr].String(), balanceChange.To.String())
-		}
-
-		fm.available[addr] = balanceChange.To
-	}
-	fm.lk.Unlock()
-	return true, nil
-}
-
-func (fm *FundMgr) getAddresses() []address.Address {
-	fm.lk.RLock()
-	defer fm.lk.RUnlock()
-	addrs := make([]address.Address, 0, len(fm.available))
-	for addr := range fm.available {
-		addrs = append(addrs, addr)
-	}
-	return addrs
-}
-
-// EnsureAvailable looks at the available balance in escrow for a given
-// address, and if less than the passed in amount, adds the difference
-func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr, wallet address.Address, amt types.BigInt) (cid.Cid, error) {
-	idAddr, err := fm.api.StateLookupID(ctx, addr, types.EmptyTSK)
-	if err != nil {
-		return cid.Undef, err
-	}
-	fm.lk.Lock()
-	defer fm.lk.Unlock()
-
-	bal, err := fm.api.StateMarketBalance(ctx, addr, types.EmptyTSK)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	stateAvail := types.BigSub(bal.Escrow, bal.Locked)
-
-	avail, ok := fm.available[idAddr]
-	if !ok {
-		avail = stateAvail
-	}
-
-	toAdd := types.BigSub(amt, avail)
-	if toAdd.LessThan(types.NewInt(0)) {
-		toAdd = types.NewInt(0)
-	}
-	fm.available[idAddr] = big.Add(avail, toAdd)
-
-	log.Infof("Funds operation w/ Expected Balance: %s, In State: %s, Requested: %s, Adding: %s", avail.String(), stateAvail.String(), amt.String(), toAdd.String())
-
-	if toAdd.LessThanEqual(big.Zero()) {
-		return cid.Undef, nil
-	}
-
-	params, err := actors.SerializeParams(&addr)
-	if err != nil {
-		fm.available[idAddr] = avail
-		return cid.Undef, err
-	}
-
-	smsg, err := fm.api.MpoolPushMessage(ctx, &types.Message{
-		To:     market.Address,
-		From:   wallet,
-		Value:  toAdd,
-		Method: market.Methods.AddBalance,
-		Params: params,
-	}, nil)
-	if err != nil {
-		fm.available[idAddr] = avail
-		return cid.Undef, err
-	}
-
-	return smsg.Cid(), nil
-}

chain/market/fundmgr_test.go

@@ -1,199 +0,0 @@
-package market
-
-import (
-	"context"
-	"errors"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-
-	tutils "github.com/filecoin-project/specs-actors/v2/support/testing"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type fakeAPI struct {
-	returnedBalance    api.MarketBalance
-	returnedBalanceErr error
-	signature          crypto.Signature
-	receivedMessage    *types.Message
-	pushMessageErr     error
-	lookupIDErr        error
-}
-
-func (fapi *fakeAPI) StateLookupID(_ context.Context, addr address.Address, _ types.TipSetKey) (address.Address, error) {
-	return addr, fapi.lookupIDErr
-}
-func (fapi *fakeAPI) StateMarketBalance(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error) {
-	return fapi.returnedBalance, fapi.returnedBalanceErr
-}
-
-func (fapi *fakeAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
-	fapi.receivedMessage = msg
-	return &types.SignedMessage{
-		Message:   *msg,
-		Signature: fapi.signature,
-	}, fapi.pushMessageErr
-}
-
-func addFundsMsg(toAdd abi.TokenAmount, addr address.Address, wallet address.Address) *types.Message {
-	params, _ := actors.SerializeParams(&addr)
-	return &types.Message{
-		To:     market.Address,
-		From:   wallet,
-		Value:  toAdd,
-		Method: market.Methods.AddBalance,
-		Params: params,
-	}
-}
-
-type expectedResult struct {
-	addAmt          abi.TokenAmount
-	shouldAdd       bool
-	err             error
-	cachedAvailable abi.TokenAmount
-}
-
-func TestAddFunds(t *testing.T) {
-	ctx := context.Background()
-	testCases := map[string]struct {
-		returnedBalanceErr error
-		returnedBalance    api.MarketBalance
-		addAmounts         []abi.TokenAmount
-		pushMessageErr     error
-		expectedResults    []expectedResult
-		lookupIDErr        error
-	}{
-		"succeeds, trivial case": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					addAmt:    abi.NewTokenAmount(100),
-					shouldAdd: true,
-					err:       nil,
-				},
-			},
-		},
-		"succeeds, money already present": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					shouldAdd:       false,
-					err:             nil,
-					cachedAvailable: abi.NewTokenAmount(100),
-				},
-			},
-		},
-		"succeeds, multiple adds": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(150), Locked: abi.NewTokenAmount(50)},
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100), abi.NewTokenAmount(200), abi.NewTokenAmount(250), abi.NewTokenAmount(250)},
-			expectedResults: []expectedResult{
-				{
-					shouldAdd: false,
-					err:       nil,
-				},
-				{
-					addAmt:          abi.NewTokenAmount(100),
-					shouldAdd:       true,
-					err:             nil,
-					cachedAvailable: abi.NewTokenAmount(200),
-				},
-				{
-					addAmt:          abi.NewTokenAmount(50),
-					shouldAdd:       true,
-					err:             nil,
-					cachedAvailable: abi.NewTokenAmount(250),
-				},
-				{
-					shouldAdd:       false,
-					err:             nil,
-					cachedAvailable: abi.NewTokenAmount(250),
-				},
-			},
-		},
-		"error on market balance": {
-			returnedBalanceErr: errors.New("something went wrong"),
-			addAmounts:         []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err: errors.New("something went wrong"),
-				},
-			},
-		},
-		"error on push message": {
-			returnedBalance: api.MarketBalance{Escrow: abi.NewTokenAmount(0), Locked: abi.NewTokenAmount(0)},
-			pushMessageErr:  errors.New("something went wrong"),
-			addAmounts:      []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err:             errors.New("something went wrong"),
-					cachedAvailable: abi.NewTokenAmount(0),
-				},
-			},
-		},
-		"error looking up address": {
-			lookupIDErr: errors.New("something went wrong"),
-			addAmounts:  []abi.TokenAmount{abi.NewTokenAmount(100)},
-			expectedResults: []expectedResult{
-				{
-					err: errors.New("something went wrong"),
-				},
-			},
-		},
-	}
-
-	for testCase, data := range testCases {
-		//nolint:scopelint
-		t.Run(testCase, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-			defer cancel()
-			sig := make([]byte, 100)
-			_, err := rand.Read(sig)
-			require.NoError(t, err)
-			fapi := &fakeAPI{
-				returnedBalance:    data.returnedBalance,
-				returnedBalanceErr: data.returnedBalanceErr,
-				signature: crypto.Signature{
-					Type: crypto.SigTypeUnknown,
-					Data: sig,
-				},
-				pushMessageErr: data.pushMessageErr,
-				lookupIDErr:    data.lookupIDErr,
-			}
-			fundMgr := newFundMgr(fapi)
-			addr := tutils.NewIDAddr(t, uint64(rand.Uint32()))
-			wallet := tutils.NewIDAddr(t, uint64(rand.Uint32()))
-			for i, amount := range data.addAmounts {
-				fapi.receivedMessage = nil
-				_, err := fundMgr.EnsureAvailable(ctx, addr, wallet, amount)
-				expected := data.expectedResults[i]
-				if expected.err == nil {
-					require.NoError(t, err)
-					if expected.shouldAdd {
-						expectedMessage := addFundsMsg(expected.addAmt, addr, wallet)
-						require.Equal(t, expectedMessage, fapi.receivedMessage)
-					} else {
-						require.Nil(t, fapi.receivedMessage)
-					}
-				} else {
-					require.EqualError(t, err, expected.err.Error())
-				}
-
-				if !expected.cachedAvailable.Nil() {
-					require.Equal(t, expected.cachedAvailable, fundMgr.available[addr])
-				}
-			}
-		})
-	}
-}

chain/market/store.go

@@ -0,0 +1,90 @@
+package market
+
+import (
+	"bytes"
+
+	cborrpc "github.com/filecoin-project/go-cbor-util"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	dsq "github.com/ipfs/go-datastore/query"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+const dsKeyAddr = "Addr"
+
+type Store struct {
+	ds datastore.Batching
+}
+
+func newStore(ds dtypes.MetadataDS) *Store {
+	ds = namespace.Wrap(ds, datastore.NewKey("/fundmgr/"))
+	return &Store{
+		ds: ds,
+	}
+}
+
+// save the state to the datastore
+func (ps *Store) save(state *FundedAddressState) error {
+	k := dskeyForAddr(state.Addr)
+
+	b, err := cborrpc.Dump(state)
+	if err != nil {
+		return err
+	}
+
+	return ps.ds.Put(k, b)
+}
+
+// get the state for the given address
+func (ps *Store) get(addr address.Address) (*FundedAddressState, error) {
+	k := dskeyForAddr(addr)
+
+	data, err := ps.ds.Get(k)
+	if err != nil {
+		return nil, err
+	}
+
+	var state FundedAddressState
+	err = cborrpc.ReadCborRPC(bytes.NewReader(data), &state)
+	if err != nil {
+		return nil, err
+	}
+	return &state, nil
+}
+
+// forEach calls iter with each address in the datastore
+func (ps *Store) forEach(iter func(*FundedAddressState)) error {
+	res, err := ps.ds.Query(dsq.Query{Prefix: dsKeyAddr})
+	if err != nil {
+		return err
+	}
+	defer res.Close() //nolint:errcheck
+
+	for {
+		res, ok := res.NextSync()
+		if !ok {
+			break
+		}
+
+		if res.Error != nil {
+			return err
+		}
+
+		var stored FundedAddressState
+		if err := stored.UnmarshalCBOR(bytes.NewReader(res.Value)); err != nil {
+			return err
+		}
+
+		iter(&stored)
+	}
+
+	return nil
+}
+
+// The datastore key used to identify the address state
+func dskeyForAddr(addr address.Address) datastore.Key {
+	return datastore.KeyWithNamespaces([]string{dsKeyAddr, addr.String()})
+}

chain/messagepool/messagepool.go

@@ -59,8 +59,6 @@ var MaxUntrustedActorPendingMessages = 10
 
 var MaxNonceGap = uint64(4)
 
-var DefaultMaxFee = abi.TokenAmount(types.MustParseFIL("0.007"))
-
 var (
 	ErrMessageTooBig = errors.New("message too big")
 
@@ -183,9 +181,15 @@ func ComputeMinRBF(curPrem abi.TokenAmount) abi.TokenAmount {
 	return types.BigAdd(minPrice, types.NewInt(1))
 }
 
-func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) {
+func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, maxFee abi.TokenAmount) {
 	if maxFee.Equals(big.Zero()) {
-		maxFee = DefaultMaxFee
+		mf, err := mff()
+		if err != nil {
+			log.Errorf("failed to get default max gas fee: %+v", err)
+			mf = big.Zero()
+		}
+
+		maxFee = mf
 	}
 
 	gl := types.NewInt(uint64(msg.GasLimit))

chain/messagepool/provider.go

@@ -2,6 +2,7 @@ package messagepool
 
 import (
 	"context"
+	"time"
 
 	"github.com/ipfs/go-cid"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -9,9 +10,16 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+var (
+	HeadChangeCoalesceMinDelay      = 2 * time.Second
+	HeadChangeCoalesceMaxDelay      = 6 * time.Second
+	HeadChangeCoalesceMergeInterval = time.Second
+)
+
 type Provider interface {
 	SubscribeHeadChanges(func(rev, app []*types.TipSet) error) *types.TipSet
 	PutMessage(m types.ChainMsg) (cid.Cid, error)
@@ -34,7 +42,13 @@ func NewProvider(sm *stmgr.StateManager, ps *pubsub.PubSub) Provider {
 }
 
 func (mpp *mpoolProvider) SubscribeHeadChanges(cb func(rev, app []*types.TipSet) error) *types.TipSet {
-	mpp.sm.ChainStore().SubscribeHeadChanges(cb)
+	mpp.sm.ChainStore().SubscribeHeadChanges(
+		store.WrapHeadChangeCoalescer(
+			cb,
+			HeadChangeCoalesceMinDelay,
+			HeadChangeCoalesceMaxDelay,
+			HeadChangeCoalesceMergeInterval,
+		))
 	return mpp.sm.ChainStore().GetHeaviestTipSet()
 }
 

chain/messagepool/selection.go

@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	tbig "github.com/filecoin-project/go-state-types/big"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -751,11 +752,10 @@ func (mp *MessagePool) createMessageChains(actor address.Address, mset map[uint6
 		balance = new(big.Int).Sub(balance, required)
 
 		value := m.Message.Value.Int
-		if balance.Cmp(value) >= 0 {
-			// Note: we only account for the value if the balance doesn't drop below 0
-			//       otherwise the message will fail and the miner can reap the gas rewards
-			balance = new(big.Int).Sub(balance, value)
+		if balance.Cmp(value) < 0 {
+			break
 		}
+		balance = new(big.Int).Sub(balance, value)
 
 		gasReward := mp.getGasReward(m, baseFee)
 		rewards = append(rewards, gasReward)

chain/stmgr/utils.go

@@ -614,6 +614,14 @@ func GetReturnType(ctx context.Context, sm *StateManager, to address.Address, me
 	return reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler), nil
 }
 
+func GetParamType(actCode cid.Cid, method abi.MethodNum) (cbg.CBORUnmarshaler, error) {
+	m, found := MethodsMap[actCode][method]
+	if !found {
+		return nil, fmt.Errorf("unknown method %d for actor %s", method, actCode)
+	}
+	return reflect.New(m.Params.Elem()).Interface().(cbg.CBORUnmarshaler), nil
+}
+
 func minerHasMinPower(ctx context.Context, sm *StateManager, addr address.Address, ts *types.TipSet) (bool, error) {
 	pact, err := sm.LoadActor(ctx, power.Address, ts)
 	if err != nil {

chain/store/coalescer.go

@@ -0,0 +1,214 @@
+package store
+
+import (
+	"context"
+	"time"
+
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+// WrapHeadChangeCoalescer wraps a ReorgNotifee with a head change coalescer.
+// minDelay is the minimum coalesce delay; when a head change is first received, the coalescer will
+//  wait for that long to coalesce more head changes.
+// maxDelay is the maximum coalesce delay; the coalescer will not delay delivery of a head change
+//  more than that.
+// mergeInterval is the interval that triggers additional coalesce delay; if the last head change was
+//  within the merge interval when the coalesce timer fires, then the coalesce time is extended
+//  by min delay and up to max delay total.
+func WrapHeadChangeCoalescer(fn ReorgNotifee, minDelay, maxDelay, mergeInterval time.Duration) ReorgNotifee {
+	c := NewHeadChangeCoalescer(fn, minDelay, maxDelay, mergeInterval)
+	return c.HeadChange
+}
+
+// HeadChangeCoalescer is a stateful reorg notifee which coalesces incoming head changes
+// with pending head changes to reduce state computations from head change notifications.
+type HeadChangeCoalescer struct {
+	notify ReorgNotifee
+
+	ctx    context.Context
+	cancel func()
+
+	eventq chan headChange
+
+	revert []*types.TipSet
+	apply  []*types.TipSet
+}
+
+type headChange struct {
+	revert, apply []*types.TipSet
+}
+
+// NewHeadChangeCoalescer creates a HeadChangeCoalescer.
+func NewHeadChangeCoalescer(fn ReorgNotifee, minDelay, maxDelay, mergeInterval time.Duration) *HeadChangeCoalescer {
+	ctx, cancel := context.WithCancel(context.Background())
+	c := &HeadChangeCoalescer{
+		notify: fn,
+		ctx:    ctx,
+		cancel: cancel,
+		eventq: make(chan headChange),
+	}
+
+	go c.background(minDelay, maxDelay, mergeInterval)
+
+	return c
+}
+
+// HeadChange is the ReorgNotifee callback for the stateful coalescer; it receives an incoming
+// head change and schedules dispatch of a coalesced head change in the background.
+func (c *HeadChangeCoalescer) HeadChange(revert, apply []*types.TipSet) error {
+	select {
+	case c.eventq <- headChange{revert: revert, apply: apply}:
+		return nil
+	case <-c.ctx.Done():
+		return c.ctx.Err()
+	}
+}
+
+// Close closes the coalescer and cancels the background dispatch goroutine.
+// Any further notification will result in an error.
+func (c *HeadChangeCoalescer) Close() error {
+	select {
+	case <-c.ctx.Done():
+	default:
+		c.cancel()
+	}
+
+	return nil
+}
+
+// Implementation details
+
+func (c *HeadChangeCoalescer) background(minDelay, maxDelay, mergeInterval time.Duration) {
+	var timerC <-chan time.Time
+	var first, last time.Time
+
+	for {
+		select {
+		case evt := <-c.eventq:
+			c.coalesce(evt.revert, evt.apply)
+
+			now := time.Now()
+			last = now
+			if first.IsZero() {
+				first = now
+			}
+
+			if timerC == nil {
+				timerC = time.After(minDelay)
+			}
+
+		case now := <-timerC:
+			sinceFirst := now.Sub(first)
+			sinceLast := now.Sub(last)
+
+			if sinceLast < mergeInterval && sinceFirst < maxDelay {
+				// coalesce some more
+				maxWait := maxDelay - sinceFirst
+				wait := minDelay
+				if maxWait < wait {
+					wait = maxWait
+				}
+
+				timerC = time.After(wait)
+			} else {
+				// dispatch
+				c.dispatch()
+
+				first = time.Time{}
+				last = time.Time{}
+				timerC = nil
+			}
+
+		case <-c.ctx.Done():
+			if c.revert != nil || c.apply != nil {
+				c.dispatch()
+			}
+			return
+		}
+	}
+}
+
+func (c *HeadChangeCoalescer) coalesce(revert, apply []*types.TipSet) {
+	// newly reverted tipsets cancel out with pending applys.
+	// similarly, newly applied tipsets cancel out with pending reverts.
+
+	// pending tipsets
+	pendRevert := make(map[types.TipSetKey]struct{}, len(c.revert))
+	for _, ts := range c.revert {
+		pendRevert[ts.Key()] = struct{}{}
+	}
+
+	pendApply := make(map[types.TipSetKey]struct{}, len(c.apply))
+	for _, ts := range c.apply {
+		pendApply[ts.Key()] = struct{}{}
+	}
+
+	// incoming tipsets
+	reverting := make(map[types.TipSetKey]struct{}, len(revert))
+	for _, ts := range revert {
+		reverting[ts.Key()] = struct{}{}
+	}
+
+	applying := make(map[types.TipSetKey]struct{}, len(apply))
+	for _, ts := range apply {
+		applying[ts.Key()] = struct{}{}
+	}
+
+	// coalesced revert set
+	// - pending reverts are cancelled by incoming applys
+	// - incoming reverts are cancelled by pending applys
+	newRevert := make([]*types.TipSet, 0, len(c.revert)+len(revert))
+	for _, ts := range c.revert {
+		_, cancel := applying[ts.Key()]
+		if cancel {
+			continue
+		}
+
+		newRevert = append(newRevert, ts)
+	}
+
+	for _, ts := range revert {
+		_, cancel := pendApply[ts.Key()]
+		if cancel {
+			continue
+		}
+
+		newRevert = append(newRevert, ts)
+	}
+
+	// coalesced apply set
+	// - pending applys are cancelled by incoming reverts
+	// - incoming applys are cancelled by pending reverts
+	newApply := make([]*types.TipSet, 0, len(c.apply)+len(apply))
+	for _, ts := range c.apply {
+		_, cancel := reverting[ts.Key()]
+		if cancel {
+			continue
+		}
+
+		newApply = append(newApply, ts)
+	}
+
+	for _, ts := range apply {
+		_, cancel := pendRevert[ts.Key()]
+		if cancel {
+			continue
+		}
+
+		newApply = append(newApply, ts)
+	}
+
+	// commit the coalesced sets
+	c.revert = newRevert
+	c.apply = newApply
+}
+
+func (c *HeadChangeCoalescer) dispatch() {
+	err := c.notify(c.revert, c.apply)
+	if err != nil {
+		log.Errorf("error dispatching coalesced head change notification: %s", err)
+	}
+
+	c.revert = nil
+	c.apply = nil
+}

chain/store/coalescer_test.go

@@ -0,0 +1,72 @@
+package store
+
+import (
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+)
+
+func TestHeadChangeCoalescer(t *testing.T) {
+	notif := make(chan headChange, 1)
+	c := NewHeadChangeCoalescer(func(revert, apply []*types.TipSet) error {
+		notif <- headChange{apply: apply, revert: revert}
+		return nil
+	},
+		100*time.Millisecond,
+		200*time.Millisecond,
+		10*time.Millisecond,
+	)
+	defer c.Close() //nolint
+
+	b0 := mock.MkBlock(nil, 0, 0)
+	root := mock.TipSet(b0)
+	bA := mock.MkBlock(root, 1, 1)
+	tA := mock.TipSet(bA)
+	bB := mock.MkBlock(root, 1, 2)
+	tB := mock.TipSet(bB)
+	tAB := mock.TipSet(bA, bB)
+	bC := mock.MkBlock(root, 1, 3)
+	tABC := mock.TipSet(bA, bB, bC)
+	bD := mock.MkBlock(root, 1, 4)
+	tABCD := mock.TipSet(bA, bB, bC, bD)
+	bE := mock.MkBlock(root, 1, 5)
+	tABCDE := mock.TipSet(bA, bB, bC, bD, bE)
+
+	c.HeadChange(nil, []*types.TipSet{tA})                      //nolint
+	c.HeadChange(nil, []*types.TipSet{tB})                      //nolint
+	c.HeadChange([]*types.TipSet{tA, tB}, []*types.TipSet{tAB}) //nolint
+	c.HeadChange([]*types.TipSet{tAB}, []*types.TipSet{tABC})   //nolint
+
+	change := <-notif
+
+	if len(change.revert) != 0 {
+		t.Fatalf("expected empty revert set but got %d elements", len(change.revert))
+	}
+	if len(change.apply) != 1 {
+		t.Fatalf("expected single element apply set but got %d elements", len(change.apply))
+	}
+	if change.apply[0] != tABC {
+		t.Fatalf("expected to apply tABC")
+	}
+
+	c.HeadChange([]*types.TipSet{tABC}, []*types.TipSet{tABCD})   //nolint
+	c.HeadChange([]*types.TipSet{tABCD}, []*types.TipSet{tABCDE}) //nolint
+
+	change = <-notif
+
+	if len(change.revert) != 1 {
+		t.Fatalf("expected single element revert set but got %d elements", len(change.revert))
+	}
+	if change.revert[0] != tABC {
+		t.Fatalf("expected to revert tABC")
+	}
+	if len(change.apply) != 1 {
+		t.Fatalf("expected single element apply set but got %d elements", len(change.apply))
+	}
+	if change.apply[0] != tABCDE {
+		t.Fatalf("expected to revert tABC")
+	}
+
+}

chain/store/index_test.go

@@ -31,7 +31,7 @@ func TestIndexSeeks(t *testing.T) {
 	ctx := context.TODO()
 
 	nbs := blockstore.NewTemporarySync()
-	cs := store.NewChainStore(nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
+	cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), nil, nil)
 
 	_, err = cs.Import(bytes.NewReader(gencar))
 	if err != nil {

chain/store/store.go

@@ -44,10 +44,10 @@ import (
 	"github.com/ipfs/go-datastore/query"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
-	car "github.com/ipld/go-car"
+	"github.com/ipld/go-car"
 	carutil "github.com/ipld/go-car/util"
 	cbg "github.com/whyrusleeping/cbor-gen"
-	pubsub "github.com/whyrusleeping/pubsub"
+	"github.com/whyrusleeping/pubsub"
 	"golang.org/x/xerrors"
 )
 
@@ -104,8 +104,11 @@ type HeadChangeEvt struct {
 //   1. a tipset cache
 //   2. a block => messages references cache.
 type ChainStore struct {
-	bs bstore.Blockstore
-	ds dstore.Batching
+	bs      bstore.Blockstore
+	localbs bstore.Blockstore
+	ds      dstore.Batching
+
+	localviewer bstore.Viewer
 
 	heaviestLk sync.Mutex
 	heaviest   *types.TipSet
@@ -130,7 +133,8 @@ type ChainStore struct {
 	journal  journal.Journal
 }
 
-func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
+// localbs is guaranteed to fail Get* if requested block isn't stored locally
+func NewChainStore(bs bstore.Blockstore, localbs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder, j journal.Journal) *ChainStore {
 	c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
 	tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
 	if j == nil {
@@ -138,6 +142,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB
 	}
 	cs := &ChainStore{
 		bs:       bs,
+		localbs:  localbs,
 		ds:       ds,
 		bestTips: pubsub.New(64),
 		tipsets:  make(map[abi.ChainEpoch][]cid.Cid),
@@ -147,6 +152,10 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallB
 		journal:  j,
 	}
 
+	if v, ok := localbs.(bstore.Viewer); ok {
+		cs.localviewer = v
+	}
+
 	cs.evtTypes = [1]journal.EventType{
 		evtTypeHeadChange: j.RegisterEventType("sync", "head_change"),
 	}
@@ -358,10 +367,32 @@ func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipS
 		// difference between 'bootstrap sync' and 'caught up' sync, we need
 		// some other heuristic.
 		return cs.takeHeaviestTipSet(ctx, ts)
+	} else if w.Equals(heaviestW) && !ts.Equals(cs.heaviest) {
+		log.Errorw("weight draw", "currTs", cs.heaviest, "ts", ts)
 	}
 	return nil
 }
 
+// ForceHeadSilent forces a chain head tipset without triggering a reorg
+// operation.
+//
+// CAUTION: Use it only for testing, such as to teleport the chain to a
+// particular tipset to carry out a benchmark, verification, etc. on a chain
+// segment.
+func (cs *ChainStore) ForceHeadSilent(_ context.Context, ts *types.TipSet) error {
+	log.Warnf("(!!!) forcing a new head silently; only use this only for testing; new head: %s", ts)
+
+	cs.heaviestLk.Lock()
+	defer cs.heaviestLk.Unlock()
+	cs.heaviest = ts
+
+	err := cs.writeHead(ts)
+	if err != nil {
+		err = xerrors.Errorf("failed to write chain head: %s", err)
+	}
+	return err
+}
+
 type reorg struct {
 	old *types.TipSet
 	new *types.TipSet
@@ -522,12 +553,20 @@ func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) {
 // GetBlock fetches a BlockHeader with the supplied CID. It returns
 // blockstore.ErrNotFound if the block was not found in the BlockStore.
 func (cs *ChainStore) GetBlock(c cid.Cid) (*types.BlockHeader, error) {
-	sb, err := cs.bs.Get(c)
-	if err != nil {
-		return nil, err
+	if cs.localviewer == nil {
+		sb, err := cs.localbs.Get(c)
+		if err != nil {
+			return nil, err
+		}
+		return types.DecodeBlock(sb.RawData())
 	}
 
-	return types.DecodeBlock(sb.RawData())
+	var blk *types.BlockHeader
+	err := cs.localviewer.View(c, func(b []byte) (err error) {
+		blk, err = types.DecodeBlock(b)
+		return err
+	})
+	return blk, err
 }
 
 func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
@@ -772,12 +811,7 @@ func (cs *ChainStore) GetGenesis() (*types.BlockHeader, error) {
 		return nil, err
 	}
 
-	genb, err := cs.bs.Get(c)
-	if err != nil {
-		return nil, err
-	}
-
-	return types.DecodeBlock(genb.RawData())
+	return cs.GetBlock(c)
 }
 
 func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
@@ -793,23 +827,39 @@ func (cs *ChainStore) GetCMessage(c cid.Cid) (types.ChainMsg, error) {
 }
 
 func (cs *ChainStore) GetMessage(c cid.Cid) (*types.Message, error) {
-	sb, err := cs.bs.Get(c)
-	if err != nil {
-		log.Errorf("get message get failed: %s: %s", c, err)
-		return nil, err
+	if cs.localviewer == nil {
+		sb, err := cs.localbs.Get(c)
+		if err != nil {
+			log.Errorf("get message get failed: %s: %s", c, err)
+			return nil, err
+		}
+		return types.DecodeMessage(sb.RawData())
 	}
 
-	return types.DecodeMessage(sb.RawData())
+	var msg *types.Message
+	err := cs.localviewer.View(c, func(b []byte) (err error) {
+		msg, err = types.DecodeMessage(b)
+		return err
+	})
+	return msg, err
 }
 
 func (cs *ChainStore) GetSignedMessage(c cid.Cid) (*types.SignedMessage, error) {
-	sb, err := cs.bs.Get(c)
-	if err != nil {
-		log.Errorf("get message get failed: %s: %s", c, err)
-		return nil, err
+	if cs.localviewer == nil {
+		sb, err := cs.localbs.Get(c)
+		if err != nil {
+			log.Errorf("get message get failed: %s: %s", c, err)
+			return nil, err
+		}
+		return types.DecodeSignedMessage(sb.RawData())
 	}
 
-	return types.DecodeSignedMessage(sb.RawData())
+	var msg *types.SignedMessage
+	err := cs.localviewer.View(c, func(b []byte) (err error) {
+		msg, err = types.DecodeSignedMessage(b)
+		return err
+	})
+	return msg, err
 }
 
 func (cs *ChainStore) readAMTCids(root cid.Cid) ([]cid.Cid, error) {
@@ -939,7 +989,7 @@ func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
 		return mmcids.bls, mmcids.secpk, nil
 	}
 
-	cst := cbor.NewCborStore(cs.bs)
+	cst := cbor.NewCborStore(cs.localbs)
 	var msgmeta types.MsgMeta
 	if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil {
 		return nil, nil, xerrors.Errorf("failed to load msgmeta (%s): %w", mmc, err)

chain/store/store_test.go

@@ -3,6 +3,7 @@ package store_test
 import (
 	"bytes"
 	"context"
+	"io"
 	"testing"
 
 	datastore "github.com/ipfs/go-datastore"
@@ -51,19 +52,25 @@ func BenchmarkGetRandomness(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	bds, err := lr.Datastore("/chain")
+	bs, err := lr.Blockstore(repo.BlockstoreChain)
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	defer func() {
+		if c, ok := bs.(io.Closer); ok {
+			if err := c.Close(); err != nil {
+				b.Logf("WARN: failed to close blockstore: %s", err)
+			}
+		}
+	}()
+
 	mds, err := lr.Datastore("/metadata")
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	bs := blockstore.NewBlockstore(bds)
-
-	cs := store.NewChainStore(bs, mds, nil, nil)
+	cs := store.NewChainStore(bs, bs, mds, nil, nil)
 
 	b.ResetTimer()
 
@@ -97,7 +104,7 @@ func TestChainExportImport(t *testing.T) {
 	}
 
 	nbs := blockstore.NewTemporary()
-	cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil)
+	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 
 	root, err := cs.Import(buf)
 	if err != nil {
@@ -131,7 +138,7 @@ func TestChainExportImportFull(t *testing.T) {
 	}
 
 	nbs := blockstore.NewTemporary()
-	cs := store.NewChainStore(nbs, datastore.NewMapDatastore(), nil, nil)
+	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), nil, nil)
 	root, err := cs.Import(buf)
 	if err != nil {
 		t.Fatal(err)

chain/sync.go

@@ -563,15 +563,16 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error {
 		)
 	}
 
-	if syncer.store.GetHeaviestTipSet().ParentWeight().GreaterThan(maybeHead.ParentWeight()) {
+	hts := syncer.store.GetHeaviestTipSet()
+
+	if hts.ParentWeight().GreaterThan(maybeHead.ParentWeight()) {
+		return nil
+	}
+	if syncer.Genesis.Equals(maybeHead) || hts.Equals(maybeHead) {
 		return nil
 	}
 
-	if syncer.Genesis.Equals(maybeHead) || syncer.store.GetHeaviestTipSet().Equals(maybeHead) {
-		return nil
-	}
-
-	if err := syncer.collectChain(ctx, maybeHead); err != nil {
+	if err := syncer.collectChain(ctx, maybeHead, hts); err != nil {
 		span.AddAttributes(trace.StringAttribute("col_error", err.Error()))
 		span.SetStatus(trace.Status{
 			Code:    13,
@@ -1684,14 +1685,14 @@ func persistMessages(ctx context.Context, bs bstore.Blockstore, bst *exchange.Co
 //
 //  3. StageMessages: having acquired the headers and found a common tipset,
 //     we then move forward, requesting the full blocks, including the messages.
-func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error {
+func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet, hts *types.TipSet) error {
 	ctx, span := trace.StartSpan(ctx, "collectChain")
 	defer span.End()
 	ss := extractSyncState(ctx)
 
-	ss.Init(syncer.store.GetHeaviestTipSet(), ts)
+	ss.Init(hts, ts)
 
-	headers, err := syncer.collectHeaders(ctx, ts, syncer.store.GetHeaviestTipSet())
+	headers, err := syncer.collectHeaders(ctx, ts, hts)
 	if err != nil {
 		ss.Error(err)
 		return err

chain/sync_manager.go

@@ -4,30 +4,43 @@ import (
 	"context"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
+	"time"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
+
 	peer "github.com/libp2p/go-libp2p-core/peer"
 )
 
-const BootstrapPeerThreshold = 2
+var (
+	BootstrapPeerThreshold = 4
 
-var coalesceForksParents = false
+	RecentSyncBufferSize = 10
+	MaxSyncWorkers       = 5
+	SyncWorkerHistory    = 3
+
+	InitialSyncTimeThreshold = 15 * time.Minute
+
+	coalesceTipsets = false
+)
 
 func init() {
-	if os.Getenv("LOTUS_SYNC_REL_PARENT") == "yes" {
-		coalesceForksParents = true
+	coalesceTipsets = os.Getenv("LOTUS_SYNC_FORMTS_PEND") == "yes"
+
+	if bootstrapPeerThreshold := os.Getenv("LOTUS_SYNC_BOOTSTRAP_PEERS"); bootstrapPeerThreshold != "" {
+		threshold, err := strconv.Atoi(bootstrapPeerThreshold)
+		if err != nil {
+			log.Errorf("failed to parse 'LOTUS_SYNC_BOOTSTRAP_PEERS' env var: %s", err)
+		} else {
+			BootstrapPeerThreshold = threshold
+		}
 	}
 }
 
-const (
-	BSStateInit      = 0
-	BSStateSelected  = 1
-	BSStateScheduled = 2
-	BSStateComplete  = 3
-)
-
 type SyncFunc func(context.Context, *types.TipSet) error
 
 // SyncManager manages the chain synchronization process, both at bootstrap time
@@ -52,108 +65,467 @@ type SyncManager interface {
 }
 
 type syncManager struct {
-	lk        sync.Mutex
-	peerHeads map[peer.ID]*types.TipSet
+	ctx    context.Context
+	cancel func()
 
-	bssLk          sync.Mutex
-	bootstrapState int
+	workq   chan peerHead
+	statusq chan workerStatus
 
-	bspThresh int
+	nextWorker uint64
+	pend       syncBucketSet
+	deferred   syncBucketSet
+	heads      map[peer.ID]*types.TipSet
+	recent     *syncBuffer
 
-	incomingTipSets chan *types.TipSet
-	syncTargets     chan *types.TipSet
-	syncResults     chan *syncResult
+	initialSyncDone bool
 
-	syncStates []*SyncerState
+	mx    sync.Mutex
+	state map[uint64]*workerState
+
+	history  []*workerState
+	historyI int
 
-	// Normally this handler is set to `(*Syncer).Sync()`.
 	doSync func(context.Context, *types.TipSet) error
-
-	stop chan struct{}
-
-	// Sync Scheduler fields
-	activeSyncs    map[types.TipSetKey]*types.TipSet
-	syncQueue      syncBucketSet
-	activeSyncTips syncBucketSet
-	nextSyncTarget *syncTargetBucket
-	workerChan     chan *types.TipSet
 }
 
 var _ SyncManager = (*syncManager)(nil)
 
-type syncResult struct {
-	ts      *types.TipSet
-	success bool
+type peerHead struct {
+	p  peer.ID
+	ts *types.TipSet
 }
 
-const syncWorkerCount = 3
+type workerState struct {
+	id uint64
+	ts *types.TipSet
+	ss *SyncerState
+	dt time.Duration
+}
 
+type workerStatus struct {
+	id  uint64
+	err error
+}
+
+// sync manager interface
 func NewSyncManager(sync SyncFunc) SyncManager {
-	sm := &syncManager{
-		bspThresh:       1,
-		peerHeads:       make(map[peer.ID]*types.TipSet),
-		syncTargets:     make(chan *types.TipSet),
-		syncResults:     make(chan *syncResult),
-		syncStates:      make([]*SyncerState, syncWorkerCount),
-		incomingTipSets: make(chan *types.TipSet),
-		activeSyncs:     make(map[types.TipSetKey]*types.TipSet),
-		doSync:          sync,
-		stop:            make(chan struct{}),
+	ctx, cancel := context.WithCancel(context.Background())
+	return &syncManager{
+		ctx:    ctx,
+		cancel: cancel,
+
+		workq:   make(chan peerHead),
+		statusq: make(chan workerStatus),
+
+		heads:   make(map[peer.ID]*types.TipSet),
+		state:   make(map[uint64]*workerState),
+		recent:  newSyncBuffer(RecentSyncBufferSize),
+		history: make([]*workerState, SyncWorkerHistory),
+
+		doSync: sync,
 	}
-	for i := range sm.syncStates {
-		sm.syncStates[i] = new(SyncerState)
-	}
-	return sm
 }
 
 func (sm *syncManager) Start() {
-	go sm.syncScheduler()
-	for i := 0; i < syncWorkerCount; i++ {
-		go sm.syncWorker(i)
-	}
+	go sm.scheduler()
 }
 
 func (sm *syncManager) Stop() {
-	close(sm.stop)
+	select {
+	case <-sm.ctx.Done():
+	default:
+		sm.cancel()
+	}
 }
 
 func (sm *syncManager) SetPeerHead(ctx context.Context, p peer.ID, ts *types.TipSet) {
-	sm.lk.Lock()
-	defer sm.lk.Unlock()
-	sm.peerHeads[p] = ts
-
-	if sm.getBootstrapState() == BSStateInit {
-		spc := sm.syncedPeerCount()
-		if spc >= sm.bspThresh {
-			// Its go time!
-			target, err := sm.selectSyncTarget()
-			if err != nil {
-				log.Error("failed to select sync target: ", err)
-				return
-			}
-			sm.setBootstrapState(BSStateSelected)
-
-			sm.incomingTipSets <- target
-		}
-		log.Infof("sync bootstrap has %d peers", spc)
-		return
+	select {
+	case sm.workq <- peerHead{p: p, ts: ts}:
+	case <-sm.ctx.Done():
+	case <-ctx.Done():
 	}
-
-	sm.incomingTipSets <- ts
 }
 
 func (sm *syncManager) State() []SyncerStateSnapshot {
-	ret := make([]SyncerStateSnapshot, 0, len(sm.syncStates))
-	for _, s := range sm.syncStates {
-		ret = append(ret, s.Snapshot())
+	sm.mx.Lock()
+	workerStates := make([]*workerState, 0, len(sm.state)+len(sm.history))
+	for _, ws := range sm.state {
+		workerStates = append(workerStates, ws)
 	}
-	return ret
+	for _, ws := range sm.history {
+		if ws != nil {
+			workerStates = append(workerStates, ws)
+		}
+	}
+	sm.mx.Unlock()
+
+	sort.Slice(workerStates, func(i, j int) bool {
+		return workerStates[i].id < workerStates[j].id
+	})
+
+	result := make([]SyncerStateSnapshot, 0, len(workerStates))
+	for _, ws := range workerStates {
+		result = append(result, ws.ss.Snapshot())
+	}
+
+	return result
 }
 
+// sync manager internals
+func (sm *syncManager) scheduler() {
+	ticker := time.NewTicker(time.Minute)
+	tickerC := ticker.C
+	for {
+		select {
+		case head := <-sm.workq:
+			sm.handlePeerHead(head)
+		case status := <-sm.statusq:
+			sm.handleWorkerStatus(status)
+		case <-tickerC:
+			if sm.initialSyncDone {
+				ticker.Stop()
+				tickerC = nil
+				sm.handleInitialSyncDone()
+			}
+		case <-sm.ctx.Done():
+			return
+		}
+	}
+}
+
+func (sm *syncManager) handlePeerHead(head peerHead) {
+	log.Infof("new peer head: %s %s", head.p, head.ts)
+
+	// have we started syncing yet?
+	if sm.nextWorker == 0 {
+		// track the peer head until we start syncing
+		sm.heads[head.p] = head.ts
+
+		// not yet; do we have enough peers?
+		if len(sm.heads) < BootstrapPeerThreshold {
+			// not enough peers; track it and wait
+			return
+		}
+
+		// we are ready to start syncing; select the sync target and spawn a worker
+		target, err := sm.selectInitialSyncTarget()
+		if err != nil {
+			log.Errorf("failed to select initial sync target: %s", err)
+			return
+		}
+
+		log.Infof("selected initial sync target: %s", target)
+		sm.spawnWorker(target)
+		return
+	}
+
+	// we have started syncing, add peer head to the queue if applicable and maybe spawn a worker
+	// if there is work to do (possibly in a fork)
+	target, work, err := sm.addSyncTarget(head.ts)
+	if err != nil {
+		log.Warnf("failed to add sync target: %s", err)
+		return
+	}
+
+	if work {
+		log.Infof("selected sync target: %s", target)
+		sm.spawnWorker(target)
+	}
+}
+
+func (sm *syncManager) handleWorkerStatus(status workerStatus) {
+	log.Debugf("worker %d done; status error: %s", status.id, status.err)
+
+	sm.mx.Lock()
+	ws := sm.state[status.id]
+	delete(sm.state, status.id)
+
+	// we track the last few workers for debug purposes
+	sm.history[sm.historyI] = ws
+	sm.historyI++
+	sm.historyI %= len(sm.history)
+	sm.mx.Unlock()
+
+	if status.err != nil {
+		// we failed to sync this target -- log it and try to work on an extended chain
+		// if there is nothing related to be worked on, we stop working on this chain.
+		log.Errorf("error during sync in %s: %s", ws.ts, status.err)
+	} else {
+		// add to the recently synced buffer
+		sm.recent.Push(ws.ts)
+		// if we are still in initial sync and this was fast enough, mark the end of the initial sync
+		if !sm.initialSyncDone && ws.dt < InitialSyncTimeThreshold {
+			sm.initialSyncDone = true
+		}
+	}
+
+	// we are done with this target, select the next sync target and spawn a worker if there is work
+	// to do, because of an extension of this chain.
+	target, work, err := sm.selectSyncTarget(ws.ts)
+	if err != nil {
+		log.Warnf("failed to select sync target: %s", err)
+		return
+	}
+
+	if work {
+		log.Infof("selected sync target: %s", target)
+		sm.spawnWorker(target)
+	}
+}
+
+func (sm *syncManager) handleInitialSyncDone() {
+	// we have just finished the initial sync; spawn some additional workers in deferred syncs
+	// as needed (and up to MaxSyncWorkers) to ramp up chain sync
+	for len(sm.state) < MaxSyncWorkers {
+		target, work, err := sm.selectDeferredSyncTarget()
+		if err != nil {
+			log.Errorf("error selecting deferred sync target: %s", err)
+			return
+		}
+
+		if !work {
+			return
+		}
+
+		log.Infof("selected deferred sync target: %s", target)
+		sm.spawnWorker(target)
+	}
+}
+
+func (sm *syncManager) spawnWorker(target *types.TipSet) {
+	id := sm.nextWorker
+	sm.nextWorker++
+	ws := &workerState{
+		id: id,
+		ts: target,
+		ss: new(SyncerState),
+	}
+	ws.ss.data.WorkerID = id
+
+	sm.mx.Lock()
+	sm.state[id] = ws
+	sm.mx.Unlock()
+
+	go sm.worker(ws)
+}
+
+func (sm *syncManager) worker(ws *workerState) {
+	log.Infof("worker %d syncing in %s", ws.id, ws.ts)
+
+	start := build.Clock.Now()
+
+	ctx := context.WithValue(sm.ctx, syncStateKey{}, ws.ss)
+	err := sm.doSync(ctx, ws.ts)
+
+	ws.dt = build.Clock.Since(start)
+	log.Infof("worker %d done; took %s", ws.id, ws.dt)
+	select {
+	case sm.statusq <- workerStatus{id: ws.id, err: err}:
+	case <-sm.ctx.Done():
+	}
+}
+
+// selects the initial sync target by examining known peer heads; only called once for the initial
+// sync.
+func (sm *syncManager) selectInitialSyncTarget() (*types.TipSet, error) {
+	var buckets syncBucketSet
+
+	var peerHeads []*types.TipSet
+	for _, ts := range sm.heads {
+		peerHeads = append(peerHeads, ts)
+	}
+	// clear the map, we don't use it any longer
+	sm.heads = nil
+
+	sort.Slice(peerHeads, func(i, j int) bool {
+		return peerHeads[i].Height() < peerHeads[j].Height()
+	})
+
+	for _, ts := range peerHeads {
+		buckets.Insert(ts)
+	}
+
+	if len(buckets.buckets) > 1 {
+		log.Warn("caution, multiple distinct chains seen during head selections")
+		// TODO: we *could* refuse to sync here without user intervention.
+		// For now, just select the best cluster
+	}
+
+	return buckets.Heaviest(), nil
+}
+
+// adds a tipset to the potential sync targets; returns true if there is a a tipset to work on.
+// this could be either a restart, eg because there is no currently scheduled sync work or a worker
+// failed or a potential fork.
+func (sm *syncManager) addSyncTarget(ts *types.TipSet) (*types.TipSet, bool, error) {
+	// Note: we don't need the state lock here to access the active worker states, as the only
+	//       competing threads that may access it do so through State() which is read only.
+
+	// if we have recently synced this or any heavier tipset we just ignore it; this can happen
+	// with an empty worker set after we just finished syncing to a target
+	if sm.recent.Synced(ts) {
+		return nil, false, nil
+	}
+
+	// if the worker set is empty, we have finished syncing and were waiting for the next tipset
+	// in this case, we just return the tipset as work to be done
+	if len(sm.state) == 0 {
+		return ts, true, nil
+	}
+
+	// check if it is related to any active sync; if so insert into the pending sync queue
+	for _, ws := range sm.state {
+		if ts.Equals(ws.ts) {
+			// ignore it, we are already syncing it
+			return nil, false, nil
+		}
+
+		if ts.Parents() == ws.ts.Key() {
+			// schedule for syncing next; it's an extension of an active sync
+			sm.pend.Insert(ts)
+			return nil, false, nil
+		}
+	}
+
+	// check to see if it is related to any pending sync; if so insert it into the pending sync queue
+	if sm.pend.RelatedToAny(ts) {
+		sm.pend.Insert(ts)
+		return nil, false, nil
+	}
+
+	// it's not related to any active or pending sync; this could be a fork in which case we
+	// start a new worker to sync it, if it is *heavier* than any active or pending set;
+	// if it is not, we ignore it.
+	for _, ws := range sm.state {
+		if isHeavier(ws.ts, ts) {
+			return nil, false, nil
+		}
+	}
+
+	pendHeaviest := sm.pend.Heaviest()
+	if pendHeaviest != nil && isHeavier(pendHeaviest, ts) {
+		return nil, false, nil
+	}
+
+	// if we have not finished the initial sync or have too many workers, add it to the deferred queue;
+	// it will be processed once a worker is freed from syncing a chain (or the initial sync finishes)
+	if !sm.initialSyncDone || len(sm.state) >= MaxSyncWorkers {
+		log.Infof("deferring sync on %s", ts)
+		sm.deferred.Insert(ts)
+		return nil, false, nil
+	}
+
+	// start a new worker, seems heavy enough and unrelated to active or pending syncs
+	return ts, true, nil
+}
+
+// selects the next sync target after a worker sync has finished; returns true and a target
+// TipSet if this chain should continue to sync because there is a heavier related tipset.
+func (sm *syncManager) selectSyncTarget(done *types.TipSet) (*types.TipSet, bool, error) {
+	// we pop the related bucket and if there is any related tipset, we work on the heaviest one next
+	// if we are not already working on a heavier tipset
+	related := sm.pend.PopRelated(done)
+	if related == nil {
+		return sm.selectDeferredSyncTarget()
+	}
+
+	heaviest := related.heaviestTipSet()
+	if isHeavier(done, heaviest) {
+		return sm.selectDeferredSyncTarget()
+	}
+
+	for _, ws := range sm.state {
+		if isHeavier(ws.ts, heaviest) {
+			return sm.selectDeferredSyncTarget()
+		}
+	}
+
+	if sm.recent.Synced(heaviest) {
+		return sm.selectDeferredSyncTarget()
+	}
+
+	return heaviest, true, nil
+}
+
+// selects a deferred sync target if there is any; these are sync targets that were not related to
+// active syncs and were deferred because there were too many workers running
+func (sm *syncManager) selectDeferredSyncTarget() (*types.TipSet, bool, error) {
+deferredLoop:
+	for !sm.deferred.Empty() {
+		bucket := sm.deferred.Pop()
+		heaviest := bucket.heaviestTipSet()
+
+		if sm.recent.Synced(heaviest) {
+			// we have synced it or something heavier recently, skip it
+			continue deferredLoop
+		}
+
+		if sm.pend.RelatedToAny(heaviest) {
+			// this has converged to a pending sync, insert it to the pending queue
+			sm.pend.Insert(heaviest)
+			continue deferredLoop
+		}
+
+		for _, ws := range sm.state {
+			if ws.ts.Equals(heaviest) || isHeavier(ws.ts, heaviest) {
+				// we have converged and are already syncing it or we are syncing on something heavier
+				// ignore it and pop the next deferred bucket
+				continue deferredLoop
+			}
+
+			if heaviest.Parents() == ws.ts.Key() {
+				// we have converged and we are syncing its parent; insert it to the pending queue
+				sm.pend.Insert(heaviest)
+				continue deferredLoop
+			}
+
+			// it's not related to any active or pending sync and this worker is free, so sync it!
+			return heaviest, true, nil
+		}
+	}
+
+	return nil, false, nil
+}
+
+func isHeavier(a, b *types.TipSet) bool {
+	return a.ParentWeight().GreaterThan(b.ParentWeight())
+}
+
+// sync buffer -- this is a circular buffer of recently synced tipsets
+type syncBuffer struct {
+	buf  []*types.TipSet
+	next int
+}
+
+func newSyncBuffer(size int) *syncBuffer {
+	return &syncBuffer{buf: make([]*types.TipSet, size)}
+}
+
+func (sb *syncBuffer) Push(ts *types.TipSet) {
+	sb.buf[sb.next] = ts
+	sb.next++
+	sb.next %= len(sb.buf)
+}
+
+func (sb *syncBuffer) Synced(ts *types.TipSet) bool {
+	for _, rts := range sb.buf {
+		if rts != nil && (rts.Equals(ts) || isHeavier(rts, ts)) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// sync buckets and related utilities
 type syncBucketSet struct {
 	buckets []*syncTargetBucket
 }
 
+type syncTargetBucket struct {
+	tips []*types.TipSet
+}
+
 func newSyncTargetBucket(tipsets ...*types.TipSet) *syncTargetBucket {
 	var stb syncTargetBucket
 	for _, ts := range tipsets {
@@ -250,10 +622,6 @@ func (sbs *syncBucketSet) Empty() bool {
 	return len(sbs.buckets) == 0
 }
 
-type syncTargetBucket struct {
-	tips []*types.TipSet
-}
-
 func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool {
 	for _, t := range stb.tips {
 		if ts.Equals(t) {
@@ -265,19 +633,43 @@ func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool {
 		if ts.Parents() == t.Key() {
 			return true
 		}
-		if coalesceForksParents && ts.Parents() == t.Parents() {
-			return true
-		}
 	}
 	return false
 }
 
 func (stb *syncTargetBucket) add(ts *types.TipSet) {
-
-	for _, t := range stb.tips {
+	for i, t := range stb.tips {
 		if t.Equals(ts) {
 			return
 		}
+		if coalesceTipsets && t.Height() == ts.Height() &&
+			types.CidArrsEqual(t.Blocks()[0].Parents, ts.Blocks()[0].Parents) {
+			miners := make(map[address.Address]struct{})
+			newTs := []*types.BlockHeader{}
+			for _, b := range t.Blocks() {
+				_, have := miners[b.Miner]
+				if !have {
+					newTs = append(newTs, b)
+					miners[b.Miner] = struct{}{}
+				}
+			}
+			for _, b := range ts.Blocks() {
+				_, have := miners[b.Miner]
+				if !have {
+					newTs = append(newTs, b)
+					miners[b.Miner] = struct{}{}
+				}
+			}
+
+			ts2, err := types.NewTipSet(newTs)
+			if err != nil {
+				log.Warnf("error while trying to recombine a tipset in a bucket: %+v", err)
+				continue
+			}
+			stb.tips[i] = ts2
+			return
+		}
+
 	}
 
 	stb.tips = append(stb.tips, ts)
@@ -296,196 +688,3 @@ func (stb *syncTargetBucket) heaviestTipSet() *types.TipSet {
 	}
 	return best
 }
-
-func (sm *syncManager) selectSyncTarget() (*types.TipSet, error) {
-	var buckets syncBucketSet
-
-	var peerHeads []*types.TipSet
-	for _, ts := range sm.peerHeads {
-		peerHeads = append(peerHeads, ts)
-	}
-	sort.Slice(peerHeads, func(i, j int) bool {
-		return peerHeads[i].Height() < peerHeads[j].Height()
-	})
-
-	for _, ts := range peerHeads {
-		buckets.Insert(ts)
-	}
-
-	if len(buckets.buckets) > 1 {
-		log.Warn("caution, multiple distinct chains seen during head selections")
-		// TODO: we *could* refuse to sync here without user intervention.
-		// For now, just select the best cluster
-	}
-
-	return buckets.Heaviest(), nil
-}
-
-func (sm *syncManager) syncScheduler() {
-	for {
-		select {
-		case ts, ok := <-sm.incomingTipSets:
-			if !ok {
-				log.Info("shutting down sync scheduler")
-				return
-			}
-
-			sm.scheduleIncoming(ts)
-		case res := <-sm.syncResults:
-			sm.scheduleProcessResult(res)
-		case sm.workerChan <- sm.nextSyncTarget.heaviestTipSet():
-			sm.scheduleWorkSent()
-		case <-sm.stop:
-			log.Info("sync scheduler shutting down")
-			return
-		}
-	}
-}
-
-func (sm *syncManager) scheduleIncoming(ts *types.TipSet) {
-	log.Debug("scheduling incoming tipset sync: ", ts.Cids())
-	if sm.getBootstrapState() == BSStateSelected {
-		sm.setBootstrapState(BSStateScheduled)
-		sm.syncTargets <- ts
-		return
-	}
-
-	var relatedToActiveSync bool
-	for _, acts := range sm.activeSyncs {
-		if ts.Equals(acts) {
-			// ignore, we are already syncing it
-			return
-		}
-
-		if ts.Parents() == acts.Key() {
-			// sync this next, after that sync process finishes
-			relatedToActiveSync = true
-		}
-	}
-
-	if !relatedToActiveSync && sm.activeSyncTips.RelatedToAny(ts) {
-		relatedToActiveSync = true
-	}
-
-	// if this is related to an active sync process, immediately bucket it
-	// we don't want to start a parallel sync process that duplicates work
-	if relatedToActiveSync {
-		sm.activeSyncTips.Insert(ts)
-		return
-	}
-
-	if sm.getBootstrapState() == BSStateScheduled {
-		sm.syncQueue.Insert(ts)
-		return
-	}
-
-	if sm.nextSyncTarget != nil && sm.nextSyncTarget.sameChainAs(ts) {
-		sm.nextSyncTarget.add(ts)
-	} else {
-		sm.syncQueue.Insert(ts)
-
-		if sm.nextSyncTarget == nil {
-			sm.nextSyncTarget = sm.syncQueue.Pop()
-			sm.workerChan = sm.syncTargets
-		}
-	}
-}
-
-func (sm *syncManager) scheduleProcessResult(res *syncResult) {
-	if res.success && sm.getBootstrapState() != BSStateComplete {
-		sm.setBootstrapState(BSStateComplete)
-	}
-
-	delete(sm.activeSyncs, res.ts.Key())
-	relbucket := sm.activeSyncTips.PopRelated(res.ts)
-	if relbucket != nil {
-		if res.success {
-			if sm.nextSyncTarget == nil {
-				sm.nextSyncTarget = relbucket
-				sm.workerChan = sm.syncTargets
-			} else {
-				for _, t := range relbucket.tips {
-					sm.syncQueue.Insert(t)
-				}
-			}
-			return
-		}
-		// TODO: this is the case where we try to sync a chain, and
-		// fail, and we have more blocks on top of that chain that
-		// have come in since.  The question is, should we try to
-		// sync these? or just drop them?
-		log.Error("failed to sync chain but have new unconnected blocks from chain")
-	}
-
-	if sm.nextSyncTarget == nil && !sm.syncQueue.Empty() {
-		next := sm.syncQueue.Pop()
-		if next != nil {
-			sm.nextSyncTarget = next
-			sm.workerChan = sm.syncTargets
-		}
-	}
-}
-
-func (sm *syncManager) scheduleWorkSent() {
-	hts := sm.nextSyncTarget.heaviestTipSet()
-	sm.activeSyncs[hts.Key()] = hts
-
-	if !sm.syncQueue.Empty() {
-		sm.nextSyncTarget = sm.syncQueue.Pop()
-	} else {
-		sm.nextSyncTarget = nil
-		sm.workerChan = nil
-	}
-}
-
-func (sm *syncManager) syncWorker(id int) {
-	ss := sm.syncStates[id]
-	for {
-		select {
-		case ts, ok := <-sm.syncTargets:
-			if !ok {
-				log.Info("sync manager worker shutting down")
-				return
-			}
-
-			ctx := context.WithValue(context.TODO(), syncStateKey{}, ss)
-			err := sm.doSync(ctx, ts)
-			if err != nil {
-				log.Errorf("sync error: %+v", err)
-			}
-
-			sm.syncResults <- &syncResult{
-				ts:      ts,
-				success: err == nil,
-			}
-		}
-	}
-}
-
-func (sm *syncManager) syncedPeerCount() int {
-	var count int
-	for _, ts := range sm.peerHeads {
-		if ts.Height() > 0 {
-			count++
-		}
-	}
-	return count
-}
-
-func (sm *syncManager) getBootstrapState() int {
-	sm.bssLk.Lock()
-	defer sm.bssLk.Unlock()
-	return sm.bootstrapState
-}
-
-func (sm *syncManager) setBootstrapState(v int) {
-	sm.bssLk.Lock()
-	defer sm.bssLk.Unlock()
-	sm.bootstrapState = v
-}
-
-func (sm *syncManager) IsBootstrapped() bool {
-	sm.bssLk.Lock()
-	defer sm.bssLk.Unlock()
-	return sm.bootstrapState == BSStateComplete
-}

chain/sync_manager_test.go

@@ -10,6 +10,10 @@ import (
 	"github.com/filecoin-project/lotus/chain/types/mock"
 )
 
+func init() {
+	BootstrapPeerThreshold = 1
+}
+
 var genTs = mock.TipSet(mock.MkBlock(nil, 0, 0))
 
 type syncOp struct {
@@ -28,7 +32,12 @@ func runSyncMgrTest(t *testing.T, tname string, thresh int, tf func(*testing.T,
 		<-ch
 		return nil
 	}).(*syncManager)
-	sm.bspThresh = thresh
+
+	oldBootstrapPeerThreshold := BootstrapPeerThreshold
+	BootstrapPeerThreshold = thresh
+	defer func() {
+		BootstrapPeerThreshold = oldBootstrapPeerThreshold
+	}()
 
 	sm.Start()
 	defer sm.Stop()
@@ -87,49 +96,59 @@ func TestSyncManagerEdgeCase(t *testing.T) {
 
 	runSyncMgrTest(t, "edgeCase", 1, func(t *testing.T, sm *syncManager, stc chan *syncOp) {
 		sm.SetPeerHead(ctx, "peer1", a)
-		assertGetSyncOp(t, stc, a)
 
 		sm.SetPeerHead(ctx, "peer1", b1)
 		sm.SetPeerHead(ctx, "peer1", b2)
-		// b1 and b2 are being processed
 
-		b1op := <-stc
-		b2op := <-stc
-		if !b1op.ts.Equals(b1) {
-			b1op, b2op = b2op, b1op
+		assertGetSyncOp(t, stc, a)
+
+		// b1 and b2 are in queue after a; the sync manager should pick the heaviest one which is b2
+		bop := <-stc
+		if !bop.ts.Equals(b2) {
+			t.Fatalf("Expected tipset %s to sync, but got %s", b2, bop.ts)
 		}
 
-		sm.SetPeerHead(ctx, "peer2", c2) // c2 is put into activeSyncTips at index 0
-		sm.SetPeerHead(ctx, "peer2", c1) // c1 is put into activeSyncTips at index 1
-		sm.SetPeerHead(ctx, "peer3", b2) // b2 is related to c2 and even though it is actively synced it is put into activeSyncTips index 0
-		sm.SetPeerHead(ctx, "peer1", a)  // a is related to b2 and is put into activeSyncTips index 0
+		sm.SetPeerHead(ctx, "peer2", c2)
+		sm.SetPeerHead(ctx, "peer2", c1)
+		sm.SetPeerHead(ctx, "peer3", b2)
+		sm.SetPeerHead(ctx, "peer1", a)
 
-		b1op.done() // b1 completes first, is related to a, so it pops activeSyncTips index 0
-		// even though correct one is index 1
+		bop.done()
 
-		b2op.done()
-		// b2 completes and is not related to c1, so it leaves activeSyncTips as it is
+		// get the next sync target; it should be c1 as the heaviest tipset but added last (same weight as c2)
+		bop = <-stc
+		if !bop.ts.Equals(c1) {
+			t.Fatalf("Expected tipset %s to sync, but got %s", c1, bop.ts)
+		}
 
-		waitUntilAllWorkersAreDone(stc)
+		sm.SetPeerHead(ctx, "peer4", d1)
+		sm.SetPeerHead(ctx, "peer5", e1)
+		bop.done()
 
-		if len(sm.activeSyncTips.buckets) != 0 {
-			t.Errorf("activeSyncTips expected empty but got: %s", sm.activeSyncTips.String())
+		// get the last sync target; it should be e1
+		var last *types.TipSet
+		for i := 0; i < 10; {
+			select {
+			case bop = <-stc:
+				bop.done()
+				if last == nil || bop.ts.Height() > last.Height() {
+					last = bop.ts
+				}
+			default:
+				i++
+				time.Sleep(10 * time.Millisecond)
+			}
+		}
+		if !last.Equals(e1) {
+			t.Fatalf("Expected tipset %s to sync, but got %s", e1, last)
+		}
+
+		if len(sm.state) != 0 {
+			t.Errorf("active syncs expected empty but got: %d", len(sm.state))
 		}
 	})
 }
 
-func waitUntilAllWorkersAreDone(stc chan *syncOp) {
-	for i := 0; i < 10; {
-		select {
-		case so := <-stc:
-			so.done()
-		default:
-			i++
-			time.Sleep(10 * time.Millisecond)
-		}
-	}
-}
-
 func TestSyncManager(t *testing.T) {
 	ctx := context.Background()
 

chain/syncstate.go

@@ -12,13 +12,14 @@ import (
 )
 
 type SyncerStateSnapshot struct {
-	Target  *types.TipSet
-	Base    *types.TipSet
-	Stage   api.SyncStateStage
-	Height  abi.ChainEpoch
-	Message string
-	Start   time.Time
-	End     time.Time
+	WorkerID uint64
+	Target   *types.TipSet
+	Base     *types.TipSet
+	Stage    api.SyncStateStage
+	Height   abi.ChainEpoch
+	Message  string
+	Start    time.Time
+	End      time.Time
 }
 
 type SyncerState struct {

chain/types/fil.go

@@ -61,6 +61,10 @@ func ParseFIL(s string) (FIL, error) {
 		}
 	}
 
+	if len(s) > 50 {
+		return FIL{}, fmt.Errorf("string length too large: %d", len(s))
+	}
+
 	r, ok := new(big.Rat).SetString(s)
 	if !ok {
 		return FIL{}, fmt.Errorf("failed to parse %q as a decimal number", s)

chain/vm/gas.go

@@ -30,7 +30,7 @@ type GasCharge struct {
 }
 
 func (g GasCharge) Total() int64 {
-	return g.ComputeGas*GasComputeMulti + g.StorageGas*GasStorageMulti
+	return g.ComputeGas + g.StorageGas
 }
 func (g GasCharge) WithVirtual(compute, storage int64) GasCharge {
 	out := g
@@ -85,6 +85,9 @@ type Pricelist interface {
 
 var prices = map[abi.ChainEpoch]Pricelist{
 	abi.ChainEpoch(0): &pricelistV0{
+		computeGasMulti: 1,
+		storageGasMulti: 1000,
+
 		onChainMessageComputeBase:    38863,
 		onChainMessageStorageBase:    36,
 		onChainMessageStoragePerByte: 1,
@@ -126,6 +129,7 @@ var prices = map[abi.ChainEpoch]Pricelist{
 				scale: 85639,
 			},
 		},
+		verifyPostDiscount:   true,
 		verifyConsensusFault: 495422,
 	},
 }

chain/vm/gas_v0.go

@@ -18,6 +18,8 @@ type scalingCost struct {
 }
 
 type pricelistV0 struct {
+	computeGasMulti int64
+	storageGasMulti int64
 	///////////////////////////////////////////////////////////////////////////
 	// System operations
 	///////////////////////////////////////////////////////////////////////////
@@ -90,6 +92,7 @@ type pricelistV0 struct {
 	computeUnsealedSectorCidBase int64
 	verifySealBase               int64
 	verifyPostLookup             map[abi.RegisteredPoStProof]scalingCost
+	verifyPostDiscount           bool
 	verifyConsensusFault         int64
 }
 
@@ -98,12 +101,12 @@ var _ Pricelist = (*pricelistV0)(nil)
 // OnChainMessage returns the gas used for storing a message of a given size in the chain.
 func (pl *pricelistV0) OnChainMessage(msgSize int) GasCharge {
 	return newGasCharge("OnChainMessage", pl.onChainMessageComputeBase,
-		pl.onChainMessageStorageBase+pl.onChainMessageStoragePerByte*int64(msgSize))
+		(pl.onChainMessageStorageBase+pl.onChainMessageStoragePerByte*int64(msgSize))*pl.storageGasMulti)
 }
 
 // OnChainReturnValue returns the gas used for storing the response of a message in the chain.
 func (pl *pricelistV0) OnChainReturnValue(dataSize int) GasCharge {
-	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte)
+	return newGasCharge("OnChainReturnValue", 0, int64(dataSize)*pl.onChainReturnValuePerByte*pl.storageGasMulti)
 }
 
 // OnMethodInvocation returns the gas used when invoking a method.
@@ -135,18 +138,18 @@ func (pl *pricelistV0) OnIpldGet() GasCharge {
 
 // OnIpldPut returns the gas used for storing an object
 func (pl *pricelistV0) OnIpldPut(dataSize int) GasCharge {
-	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte).
+	return newGasCharge("OnIpldPut", pl.ipldPutBase, int64(dataSize)*pl.ipldPutPerByte*pl.storageGasMulti).
 		WithExtra(dataSize)
 }
 
 // OnCreateActor returns the gas used for creating an actor
 func (pl *pricelistV0) OnCreateActor() GasCharge {
-	return newGasCharge("OnCreateActor", pl.createActorCompute, pl.createActorStorage)
+	return newGasCharge("OnCreateActor", pl.createActorCompute, pl.createActorStorage*pl.storageGasMulti)
 }
 
 // OnDeleteActor returns the gas used for deleting an actor
 func (pl *pricelistV0) OnDeleteActor() GasCharge {
-	return newGasCharge("OnDeleteActor", 0, pl.deleteActor)
+	return newGasCharge("OnDeleteActor", 0, pl.deleteActor*pl.storageGasMulti)
 }
 
 // OnVerifySignature
@@ -201,7 +204,9 @@ func (pl *pricelistV0) OnVerifyPost(info proof2.WindowPoStVerifyInfo) GasCharge
 	}
 
 	gasUsed := cost.flat + int64(len(info.ChallengedSectors))*cost.scale
-	gasUsed /= 2 // XXX: this is an artificial discount
+	if pl.verifyPostDiscount {
+		gasUsed /= 2 // XXX: this is an artificial discount
+	}
 
 	return newGasCharge("OnVerifyPost", gasUsed, 0).
 		WithExtra(map[string]interface{}{

chain/vm/runtime.go

@@ -546,9 +546,10 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError
 
 	// overflow safe
 	if rt.gasUsed > rt.gasAvailable-toUse {
+		gasUsed := rt.gasUsed
 		rt.gasUsed = rt.gasAvailable
-		return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d",
-			rt.gasUsed, rt.gasAvailable)
+		return aerrors.Newf(exitcode.SysErrOutOfGas, "not enough gas: used=%d, available=%d, use=%d",
+			gasUsed, rt.gasAvailable, toUse)
 	}
 	rt.gasUsed += toUse
 	return nil

chain/vm/vm.go

@@ -70,6 +70,7 @@ func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Ad
 }
 
 var _ cbor.IpldBlockstore = (*gasChargingBlocks)(nil)
+var _ blockstore.Viewer = (*gasChargingBlocks)(nil)
 
 type gasChargingBlocks struct {
 	chargeGas func(GasCharge)
@@ -77,6 +78,24 @@ type gasChargingBlocks struct {
 	under     cbor.IpldBlockstore
 }
 
+func (bs *gasChargingBlocks) View(c cid.Cid, cb func([]byte) error) error {
+	if v, ok := bs.under.(blockstore.Viewer); ok {
+		bs.chargeGas(bs.pricelist.OnIpldGet())
+		return v.View(c, func(b []byte) error {
+			// we have successfully retrieved the value; charge for it, even if the user-provided function fails.
+			bs.chargeGas(newGasCharge("OnIpldViewEnd", 0, 0).WithExtra(len(b)))
+			bs.chargeGas(gasOnActorExec)
+			return cb(b)
+		})
+	}
+	// the underlying blockstore doesn't implement the viewer interface, fall back to normal Get behaviour.
+	blk, err := bs.Get(c)
+	if err == nil && blk != nil {
+		return cb(blk.RawData())
+	}
+	return err
+}
+
 func (bs *gasChargingBlocks) Get(c cid.Cid) (block.Block, error) {
 	bs.chargeGas(bs.pricelist.OnIpldGet())
 	blk, err := bs.under.Get(c)
@@ -130,10 +149,10 @@ func (vm *VM) makeRuntime(ctx context.Context, msg *types.Message, parent *Runti
 		rt.Abortf(exitcode.SysErrForbidden, "message execution exceeds call depth")
 	}
 
-	rt.cst = &cbor.BasicIpldStore{
-		Blocks: &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks},
-		Atlas:  vm.cst.Atlas,
-	}
+	cbb := &gasChargingBlocks{rt.chargeGasFunc(2), rt.pricelist, vm.cst.Blocks}
+	cst := cbor.NewCborStore(cbb)
+	cst.Atlas = vm.cst.Atlas // associate the atlas.
+	rt.cst = cst
 
 	vmm := *msg
 	resF, ok := rt.ResolveAddress(msg.From)

chain/wallet/multi.go

@@ -90,7 +90,7 @@ func (m MultiWallet) WalletHas(ctx context.Context, address address.Address) (bo
 }
 
 func (m MultiWallet) WalletList(ctx context.Context) ([]address.Address, error) {
-	var out []address.Address
+	out := make([]address.Address, 0)
 	seen := map[address.Address]struct{}{}
 
 	ws := nonNil(m.Remote, m.Ledger, m.Local)

chain/wallet/wallet.go

@@ -283,6 +283,10 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er
 	w.lk.Lock()
 	defer w.lk.Unlock()
 
+	if err := w.keystore.Delete(KTrashPrefix + k.Address.String()); err != nil && !xerrors.Is(err, types.ErrKeyInfoNotFound) {
+		return xerrors.Errorf("failed to purge trashed key %s: %w", addr, err)
+	}
+
 	if err := w.keystore.Put(KTrashPrefix+k.Address.String(), k.KeyInfo); err != nil {
 		return xerrors.Errorf("failed to mark key %s as trashed: %w", addr, err)
 	}
@@ -301,6 +305,18 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er
 
 	delete(w.keys, addr)
 
+	def, err := w.GetDefault()
+	if err != nil {
+		return xerrors.Errorf("getting default address: %w", err)
+	}
+
+	if def == addr {
+		err = w.SetDefault(address.Undef)
+		if err != nil {
+			return xerrors.Errorf("unsetting default address: %w", err)
+		}
+	}
+
 	return nil
 }
 

cli/client.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -81,9 +82,11 @@ var clientCmd = &cli.Command{
 		WithCategory("storage", clientListDeals),
 		WithCategory("storage", clientGetDealCmd),
 		WithCategory("storage", clientListAsksCmd),
+		WithCategory("storage", clientDealStatsCmd),
 		WithCategory("data", clientImportCmd),
 		WithCategory("data", clientDropCmd),
 		WithCategory("data", clientLocalCmd),
+		WithCategory("data", clientStat),
 		WithCategory("retrieval", clientFindCmd),
 		WithCategory("retrieval", clientRetrieveCmd),
 		WithCategory("util", clientCommPCmd),
@@ -91,6 +94,7 @@ var clientCmd = &cli.Command{
 		WithCategory("util", clientInfoCmd),
 		WithCategory("util", clientListTransfers),
 		WithCategory("util", clientRestartTransfer),
+		WithCategory("util", clientCancelTransfer),
 	},
 }
 
@@ -491,7 +495,7 @@ func interactiveDeal(cctx *cli.Context) error {
 	var dur time.Duration
 	var epochs abi.ChainEpoch
 	var verified bool
-	var ds lapi.DataSize
+	var ds lapi.DataCIDSize
 
 	// find
 	var candidateAsks []*storagemarket.StorageAsk
@@ -555,7 +559,7 @@ uiLoop:
 			}
 
 			color.Blue(".. calculating data size\n")
-			ds, err = api.ClientDealSize(ctx, data)
+			ds, err = api.ClientDealPieceCID(ctx, data)
 			if err != nil {
 				return err
 			}
@@ -845,6 +849,9 @@ uiLoop:
 					Data: &storagemarket.DataRef{
 						TransferType: storagemarket.TTGraphsync,
 						Root:         data,
+
+						PieceCid:  &ds.PieceCID,
+						PieceSize: ds.PieceSize.Unpadded(),
 					},
 					Wallet:            a,
 					Miner:             maddr,
@@ -1107,6 +1114,80 @@ var clientRetrieveCmd = &cli.Command{
 	},
 }
 
+var clientDealStatsCmd = &cli.Command{
+	Name:  "deal-stats",
+	Usage: "Print statistics about local storage deals",
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name: "newer-than",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		localDeals, err := api.ClientListDeals(ctx)
+		if err != nil {
+			return err
+		}
+
+		var totalSize uint64
+		byState := map[storagemarket.StorageDealStatus][]uint64{}
+		for _, deal := range localDeals {
+			if cctx.IsSet("newer-than") {
+				if time.Now().Sub(deal.CreationTime) > cctx.Duration("newer-than") {
+					continue
+				}
+			}
+
+			totalSize += deal.Size
+			byState[deal.State] = append(byState[deal.State], deal.Size)
+		}
+
+		fmt.Printf("Total: %d deals, %s\n", len(localDeals), types.SizeStr(types.NewInt(totalSize)))
+
+		type stateStat struct {
+			state storagemarket.StorageDealStatus
+			count int
+			bytes uint64
+		}
+
+		stateStats := make([]stateStat, 0, len(byState))
+		for state, deals := range byState {
+			if state == storagemarket.StorageDealActive {
+				state = math.MaxUint64 // for sort
+			}
+
+			st := stateStat{
+				state: state,
+				count: len(deals),
+			}
+			for _, b := range deals {
+				st.bytes += b
+			}
+
+			stateStats = append(stateStats, st)
+		}
+
+		sort.Slice(stateStats, func(i, j int) bool {
+			return int64(stateStats[i].state) < int64(stateStats[j].state)
+		})
+
+		for _, st := range stateStats {
+			if st.state == math.MaxUint64 {
+				st.state = storagemarket.StorageDealActive
+			}
+			fmt.Printf("%s: %d deals, %s\n", storagemarket.DealStates[st.state], st.count, types.SizeStr(types.NewInt(st.bytes)))
+		}
+
+		return nil
+	},
+}
+
 var clientListAsksCmd = &cli.Command{
 	Name:  "list-asks",
 	Usage: "List asks for top miners",
@@ -1638,6 +1719,39 @@ var clientInfoCmd = &cli.Command{
 	},
 }
 
+var clientStat = &cli.Command{
+	Name:      "stat",
+	Usage:     "Print information about a locally stored file (piece size, etc)",
+	ArgsUsage: "<cid>",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() || cctx.NArg() != 1 {
+			return fmt.Errorf("must specify cid of data")
+		}
+
+		dataCid, err := cid.Parse(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("parsing data cid: %w", err)
+		}
+
+		ds, err := api.ClientDealSize(ctx, dataCid)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Piece Size  : %v\n", ds.PieceSize)
+		fmt.Printf("Payload Size: %v\n", ds.PayloadSize)
+
+		return nil
+	},
+}
+
 var clientRestartTransfer = &cli.Command{
 	Name:  "restart-transfer",
 	Usage: "Force restart a stalled data transfer",
@@ -1698,6 +1812,66 @@ var clientRestartTransfer = &cli.Command{
 	},
 }
 
+var clientCancelTransfer = &cli.Command{
+	Name:  "cancel-transfer",
+	Usage: "Force cancel a data transfer",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "peerid",
+			Usage: "narrow to transfer with specific peer",
+		},
+		&cli.BoolFlag{
+			Name:  "initiator",
+			Usage: "specify only transfers where peer is/is not initiator",
+			Value: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return cli.ShowCommandHelp(cctx, cctx.Command.Name)
+		}
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		transferUint, err := strconv.ParseUint(cctx.Args().First(), 10, 64)
+		if err != nil {
+			return fmt.Errorf("Error reading transfer ID: %w", err)
+		}
+		transferID := datatransfer.TransferID(transferUint)
+		initiator := cctx.Bool("initiator")
+		var other peer.ID
+		if pidstr := cctx.String("peerid"); pidstr != "" {
+			p, err := peer.Decode(pidstr)
+			if err != nil {
+				return err
+			}
+			other = p
+		} else {
+			channels, err := api.ClientListDataTransfers(ctx)
+			if err != nil {
+				return err
+			}
+			found := false
+			for _, channel := range channels {
+				if channel.IsInitiator == initiator && channel.TransferID == transferID {
+					other = channel.OtherPeer
+					found = true
+					break
+				}
+			}
+			if !found {
+				return errors.New("unable to find matching data transfer")
+			}
+		}
+
+		return api.ClientCancelDataTransfer(ctx, transferID, other, initiator)
+	},
+}
+
 var clientListTransfers = &cli.Command{
 	Name:  "list-transfers",
 	Usage: "List ongoing data transfers for deals",
@@ -1715,6 +1889,10 @@ var clientListTransfers = &cli.Command{
 			Name:  "watch",
 			Usage: "watch deal updates in real-time, rather than a one time list",
 		},
+		&cli.BoolFlag{
+			Name:  "show-failed",
+			Usage: "show failed/cancelled transfers",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -1732,7 +1910,7 @@ var clientListTransfers = &cli.Command{
 		completed := cctx.Bool("completed")
 		color := cctx.Bool("color")
 		watch := cctx.Bool("watch")
-
+		showFailed := cctx.Bool("show-failed")
 		if watch {
 			channelUpdates, err := api.ClientDataTransferUpdates(ctx)
 			if err != nil {
@@ -1744,7 +1922,7 @@ var clientListTransfers = &cli.Command{
 
 				tm.MoveCursor(1, 1)
 
-				OutputDataTransferChannels(tm.Screen, channels, completed, color)
+				OutputDataTransferChannels(tm.Screen, channels, completed, color, showFailed)
 
 				tm.Flush()
 
@@ -1769,13 +1947,13 @@ var clientListTransfers = &cli.Command{
 				}
 			}
 		}
-		OutputDataTransferChannels(os.Stdout, channels, completed, color)
+		OutputDataTransferChannels(os.Stdout, channels, completed, color, showFailed)
 		return nil
 	},
 }
 
 // OutputDataTransferChannels generates table output for a list of channels
-func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChannel, completed bool, color bool) {
+func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChannel, completed bool, color bool, showFailed bool) {
 	sort.Slice(channels, func(i, j int) bool {
 		return channels[i].TransferID < channels[j].TransferID
 	})
@@ -1785,6 +1963,9 @@ func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChann
 		if !completed && channel.Status == datatransfer.Completed {
 			continue
 		}
+		if !showFailed && (channel.Status == datatransfer.Failed || channel.Status == datatransfer.Cancelled) {
+			continue
+		}
 		if channel.IsSender {
 			sendingChannels = append(sendingChannels, channel)
 		} else {

cli/cmd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"strings"
@@ -206,7 +207,22 @@ func GetFullNodeAPI(ctx *cli.Context) (api.FullNode, jsonrpc.ClientCloser, error
 	return client.NewFullNodeRPC(ctx.Context, addr, headers)
 }
 
-func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+type GetStorageMinerOptions struct {
+	PreferHttp bool
+}
+
+type GetStorageMinerOption func(*GetStorageMinerOptions)
+
+func StorageMinerUseHttp(opts *GetStorageMinerOptions) {
+	opts.PreferHttp = true
+}
+
+func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.StorageMiner, jsonrpc.ClientCloser, error) {
+	var options GetStorageMinerOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
 	if tn, ok := ctx.App.Metadata["testnode-storage"]; ok {
 		return tn.(api.StorageMiner), func() {}, nil
 	}
@@ -216,7 +232,23 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...jsonrpc.Option) (api.StorageMi
 		return nil, nil, err
 	}
 
-	return client.NewStorageMinerRPC(ctx.Context, addr, headers, opts...)
+	if options.PreferHttp {
+		u, err := url.Parse(addr)
+		if err != nil {
+			return nil, nil, xerrors.Errorf("parsing miner api URL: %w", err)
+		}
+
+		switch u.Scheme {
+		case "ws":
+			u.Scheme = "http"
+		case "wss":
+			u.Scheme = "https"
+		}
+
+		addr = u.String()
+	}
+
+	return client.NewStorageMinerRPC(ctx.Context, addr, headers)
 }
 
 func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {

cli/mpool.go

@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/config"
 )
 
 var mpoolCmd = &cli.Command{
@@ -434,7 +435,12 @@ var mpoolReplaceCmd = &cli.Command{
 
 			msg.GasPremium = big.Max(retm.GasPremium, minRBF)
 			msg.GasFeeCap = big.Max(retm.GasFeeCap, msg.GasPremium)
-			messagepool.CapGasFee(&msg, mss.Get().MaxFee)
+
+			mff := func() (abi.TokenAmount, error) {
+				return abi.TokenAmount(config.DefaultDefaultMaxFee), nil
+			}
+
+			messagepool.CapGasFee(mff, &msg, mss.Get().MaxFee)
 		} else {
 			msg.GasLimit = cctx.Int64("gas-limit")
 			msg.GasPremium, err = types.BigFromString(cctx.String("gas-premium"))

cli/multisig.go

@@ -264,8 +264,19 @@ var msigInspectCmd = &cli.Command{
 		}
 		fmt.Fprintf(cctx.App.Writer, "Threshold: %d / %d\n", threshold, len(signers))
 		fmt.Fprintln(cctx.App.Writer, "Signers:")
+
+		signerTable := tabwriter.NewWriter(cctx.App.Writer, 8, 4, 2, ' ', 0)
+		fmt.Fprintf(signerTable, "ID\tAddress\n")
 		for _, s := range signers {
-			fmt.Fprintf(cctx.App.Writer, "\t%s\n", s)
+			signerActor, err := api.StateAccountKey(ctx, s, types.EmptyTSK)
+			if err != nil {
+				fmt.Fprintf(signerTable, "%s\t%s\n", s, "N/A")
+			} else {
+				fmt.Fprintf(signerTable, "%s\t%s\n", s, signerActor)
+			}
+		}
+		if err := signerTable.Flush(); err != nil {
+			return xerrors.Errorf("flushing output: %+v", err)
 		}
 
 		pending := make(map[int64]multisig.Transaction)
@@ -296,27 +307,33 @@ var msigInspectCmd = &cli.Command{
 					target += " (self)"
 				}
 				targAct, err := api.StateGetActor(ctx, tx.To, types.EmptyTSK)
-				if err != nil {
-					return xerrors.Errorf("failed to resolve 'To' address of multisig transaction %d: %w", txid, err)
-				}
-				method := stmgr.MethodsMap[targAct.Code][tx.Method]
-
 				paramStr := fmt.Sprintf("%x", tx.Params)
-				if decParams && tx.Method != 0 {
-					ptyp := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler)
-					if err := ptyp.UnmarshalCBOR(bytes.NewReader(tx.Params)); err != nil {
-						return xerrors.Errorf("failed to decode parameters of transaction %d: %w", txid, err)
+
+				if err != nil {
+					if tx.Method == 0 {
+						fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), "Send", tx.Method, paramStr)
+					} else {
+						fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), "new account, unknown method", tx.Method, paramStr)
+					}
+				} else {
+					method := stmgr.MethodsMap[targAct.Code][tx.Method]
+
+					if decParams && tx.Method != 0 {
+						ptyp := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler)
+						if err := ptyp.UnmarshalCBOR(bytes.NewReader(tx.Params)); err != nil {
+							return xerrors.Errorf("failed to decode parameters of transaction %d: %w", txid, err)
+						}
+
+						b, err := json.Marshal(ptyp)
+						if err != nil {
+							return xerrors.Errorf("could not json marshal parameter type: %w", err)
+						}
+
+						paramStr = string(b)
 					}
 
-					b, err := json.Marshal(ptyp)
-					if err != nil {
-						return xerrors.Errorf("could not json marshal parameter type: %w", err)
-					}
-
-					paramStr = string(b)
+					fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), method.Name, tx.Method, paramStr)
 				}
-
-				fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\t%s(%d)\t%s\n", txid, "pending", len(tx.Approved), target, types.FIL(tx.Value), method.Name, tx.Method, paramStr)
 			}
 			if err := w.Flush(); err != nil {
 				return xerrors.Errorf("flushing output: %+v", err)

cli/paych_test.go

@@ -1,9 +1,7 @@
 package cli
 
 import (
-	"bytes"
 	"context"
-	"flag"
 	"fmt"
 	"os"
 	"regexp"
@@ -12,24 +10,21 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/require"
-	"github.com/urfave/cli/v2"
+	clitest "github.com/filecoin-project/lotus/cli/test"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
-	cbor "github.com/ipfs/go-ipld-cbor"
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/lotus/api/apibstore"
 	"github.com/filecoin-project/lotus/api/test"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/types"
-	builder "github.com/filecoin-project/lotus/node/test"
 )
 
 func init() {
@@ -42,24 +37,24 @@ func init() {
 // commands
 func TestPaymentChannels(t *testing.T) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+	clitest.QuietMiningLogs()
 
 	blocktime := 5 * time.Millisecond
 	ctx := context.Background()
-	nodes, addrs := startTwoNodesOneMiner(ctx, t, blocktime)
+	nodes, addrs := clitest.StartTwoNodesOneMiner(ctx, t, blocktime)
 	paymentCreator := nodes[0]
 	paymentReceiver := nodes[1]
 	creatorAddr := addrs[0]
 	receiverAddr := addrs[1]
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t)
-	creatorCLI := mockCLI.client(paymentCreator.ListenAddr)
-	receiverCLI := mockCLI.client(paymentReceiver.ListenAddr)
+	mockCLI := clitest.NewMockCLI(ctx, t, Commands)
+	creatorCLI := mockCLI.Client(paymentCreator.ListenAddr)
+	receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr)
 
 	// creator: paych add-funds <creator> <receiver> <amount>
 	channelAmt := "100000"
-	cmd := []string{creatorAddr.String(), receiverAddr.String(), channelAmt}
-	chstr := creatorCLI.runCmd(paychAddFundsCmd, cmd)
+	chstr := creatorCLI.RunCmd("paych", "add-funds", creatorAddr.String(), receiverAddr.String(), channelAmt)
 
 	chAddr, err := address.NewFromString(chstr)
 	require.NoError(t, err)
@@ -67,16 +62,13 @@ func TestPaymentChannels(t *testing.T) {
 	// creator: paych voucher create <channel> <amount>
 	voucherAmt := 100
 	vamt := strconv.Itoa(voucherAmt)
-	cmd = []string{chAddr.String(), vamt}
-	voucher := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher := creatorCLI.RunCmd("paych", "voucher", "create", chAddr.String(), vamt)
 
 	// receiver: paych voucher add <channel> <voucher>
-	cmd = []string{chAddr.String(), voucher}
-	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "add", chAddr.String(), voucher)
 
 	// creator: paych settle <channel>
-	cmd = []string{chAddr.String()}
-	creatorCLI.runCmd(paychSettleCmd, cmd)
+	creatorCLI.RunCmd("paych", "settle", chAddr.String())
 
 	// Wait for the chain to reach the settle height
 	chState := getPaychState(ctx, t, paymentReceiver, chAddr)
@@ -85,8 +77,7 @@ func TestPaymentChannels(t *testing.T) {
 	waitForHeight(ctx, t, paymentReceiver, sa)
 
 	// receiver: paych collect <channel>
-	cmd = []string{chAddr.String()}
-	receiverCLI.runCmd(paychCloseCmd, cmd)
+	receiverCLI.RunCmd("paych", "collect", chAddr.String())
 }
 
 type voucherSpec struct {
@@ -98,20 +89,21 @@ type voucherSpec struct {
 // TestPaymentChannelStatus tests the payment channel status CLI command
 func TestPaymentChannelStatus(t *testing.T) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+	clitest.QuietMiningLogs()
 
 	blocktime := 5 * time.Millisecond
 	ctx := context.Background()
-	nodes, addrs := startTwoNodesOneMiner(ctx, t, blocktime)
+	nodes, addrs := clitest.StartTwoNodesOneMiner(ctx, t, blocktime)
 	paymentCreator := nodes[0]
 	creatorAddr := addrs[0]
 	receiverAddr := addrs[1]
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t)
-	creatorCLI := mockCLI.client(paymentCreator.ListenAddr)
+	mockCLI := clitest.NewMockCLI(ctx, t, Commands)
+	creatorCLI := mockCLI.Client(paymentCreator.ListenAddr)
 
-	cmd := []string{creatorAddr.String(), receiverAddr.String()}
-	out := creatorCLI.runCmd(paychStatusByFromToCmd, cmd)
+	// creator: paych status-by-from-to <creator> <receiver>
+	out := creatorCLI.RunCmd("paych", "status-by-from-to", creatorAddr.String(), receiverAddr.String())
 	fmt.Println(out)
 	noChannelState := "Channel does not exist"
 	require.Regexp(t, regexp.MustCompile(noChannelState), out)
@@ -120,14 +112,17 @@ func TestPaymentChannelStatus(t *testing.T) {
 	create := make(chan string)
 	go func() {
 		// creator: paych add-funds <creator> <receiver> <amount>
-		cmd := []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)}
-		create <- creatorCLI.runCmd(paychAddFundsCmd, cmd)
+		create <- creatorCLI.RunCmd(
+			"paych",
+			"add-funds",
+			creatorAddr.String(),
+			receiverAddr.String(),
+			fmt.Sprintf("%d", channelAmt))
 	}()
 
 	// Wait for the output to stop being "Channel does not exist"
 	for regexp.MustCompile(noChannelState).MatchString(out) {
-		cmd := []string{creatorAddr.String(), receiverAddr.String()}
-		out = creatorCLI.runCmd(paychStatusByFromToCmd, cmd)
+		out = creatorCLI.RunCmd("paych", "status-by-from-to", creatorAddr.String(), receiverAddr.String())
 	}
 	fmt.Println(out)
 
@@ -147,8 +142,7 @@ func TestPaymentChannelStatus(t *testing.T) {
 	// Wait for create channel to complete
 	chstr := <-create
 
-	cmd = []string{chstr}
-	out = creatorCLI.runCmd(paychStatusCmd, cmd)
+	out = creatorCLI.RunCmd("paych", "status", chstr)
 	fmt.Println(out)
 	// Output should have the channel address
 	require.Regexp(t, regexp.MustCompile("Channel.*"+chstr), out)
@@ -160,11 +154,9 @@ func TestPaymentChannelStatus(t *testing.T) {
 
 	// creator: paych voucher create <channel> <amount>
 	voucherAmt := uint64(10)
-	cmd = []string{chAddr.String(), fmt.Sprintf("%d", voucherAmt)}
-	creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	creatorCLI.RunCmd("paych", "voucher", "create", chAddr.String(), fmt.Sprintf("%d", voucherAmt))
 
-	cmd = []string{chstr}
-	out = creatorCLI.runCmd(paychStatusCmd, cmd)
+	out = creatorCLI.RunCmd("paych", "status", chstr)
 	fmt.Println(out)
 	voucherAmtAtto := types.BigMul(types.NewInt(voucherAmt), types.NewInt(build.FilecoinPrecision))
 	voucherAmtStr := fmt.Sprintf("%d", voucherAmtAtto)
@@ -176,24 +168,24 @@ func TestPaymentChannelStatus(t *testing.T) {
 // channel voucher commands
 func TestPaymentChannelVouchers(t *testing.T) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+	clitest.QuietMiningLogs()
 
 	blocktime := 5 * time.Millisecond
 	ctx := context.Background()
-	nodes, addrs := startTwoNodesOneMiner(ctx, t, blocktime)
+	nodes, addrs := clitest.StartTwoNodesOneMiner(ctx, t, blocktime)
 	paymentCreator := nodes[0]
 	paymentReceiver := nodes[1]
 	creatorAddr := addrs[0]
 	receiverAddr := addrs[1]
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t)
-	creatorCLI := mockCLI.client(paymentCreator.ListenAddr)
-	receiverCLI := mockCLI.client(paymentReceiver.ListenAddr)
+	mockCLI := clitest.NewMockCLI(ctx, t, Commands)
+	creatorCLI := mockCLI.Client(paymentCreator.ListenAddr)
+	receiverCLI := mockCLI.Client(paymentReceiver.ListenAddr)
 
 	// creator: paych add-funds <creator> <receiver> <amount>
 	channelAmt := "100000"
-	cmd := []string{creatorAddr.String(), receiverAddr.String(), channelAmt}
-	chstr := creatorCLI.runCmd(paychAddFundsCmd, cmd)
+	chstr := creatorCLI.RunCmd("paych", "add-funds", creatorAddr.String(), receiverAddr.String(), channelAmt)
 
 	chAddr, err := address.NewFromString(chstr)
 	require.NoError(t, err)
@@ -203,39 +195,33 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	// creator: paych voucher create <channel> <amount>
 	// Note: implied --lane=0
 	voucherAmt1 := 100
-	cmd = []string{chAddr.String(), strconv.Itoa(voucherAmt1)}
-	voucher1 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher1 := creatorCLI.RunCmd("paych", "voucher", "create", chAddr.String(), strconv.Itoa(voucherAmt1))
 	vouchers = append(vouchers, voucherSpec{serialized: voucher1, lane: 0, amt: voucherAmt1})
 
 	// creator: paych voucher create <channel> <amount> --lane=5
 	lane5 := "--lane=5"
 	voucherAmt2 := 50
-	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt2)}
-	voucher2 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher2 := creatorCLI.RunCmd("paych", "voucher", "create", lane5, chAddr.String(), strconv.Itoa(voucherAmt2))
 	vouchers = append(vouchers, voucherSpec{serialized: voucher2, lane: 5, amt: voucherAmt2})
 
 	// creator: paych voucher create <channel> <amount> --lane=5
 	voucherAmt3 := 70
-	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt3)}
-	voucher3 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher3 := creatorCLI.RunCmd("paych", "voucher", "create", lane5, chAddr.String(), strconv.Itoa(voucherAmt3))
 	vouchers = append(vouchers, voucherSpec{serialized: voucher3, lane: 5, amt: voucherAmt3})
 
 	// creator: paych voucher create <channel> <amount> --lane=5
 	voucherAmt4 := 80
-	cmd = []string{lane5, chAddr.String(), strconv.Itoa(voucherAmt4)}
-	voucher4 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher4 := creatorCLI.RunCmd("paych", "voucher", "create", lane5, chAddr.String(), strconv.Itoa(voucherAmt4))
 	vouchers = append(vouchers, voucherSpec{serialized: voucher4, lane: 5, amt: voucherAmt4})
 
 	// creator: paych voucher list <channel> --export
-	cmd = []string{"--export", chAddr.String()}
-	list := creatorCLI.runCmd(paychVoucherListCmd, cmd)
+	list := creatorCLI.RunCmd("paych", "voucher", "list", "--export", chAddr.String())
 
 	// Check that voucher list output is correct on creator
 	checkVoucherOutput(t, list, vouchers)
 
 	// creator: paych voucher best-spendable <channel>
-	cmd = []string{"--export", chAddr.String()}
-	bestSpendable := creatorCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+	bestSpendable := creatorCLI.RunCmd("paych", "voucher", "best-spendable", "--export", chAddr.String())
 
 	// Check that best spendable output is correct on creator
 	bestVouchers := []voucherSpec{
@@ -245,31 +231,25 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	checkVoucherOutput(t, bestSpendable, bestVouchers)
 
 	// receiver: paych voucher add <voucher>
-	cmd = []string{chAddr.String(), voucher1}
-	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "add", chAddr.String(), voucher1)
 
 	// receiver: paych voucher add <voucher>
-	cmd = []string{chAddr.String(), voucher2}
-	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "add", chAddr.String(), voucher2)
 
 	// receiver: paych voucher add <voucher>
-	cmd = []string{chAddr.String(), voucher3}
-	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "add", chAddr.String(), voucher3)
 
 	// receiver: paych voucher add <voucher>
-	cmd = []string{chAddr.String(), voucher4}
-	receiverCLI.runCmd(paychVoucherAddCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "add", chAddr.String(), voucher4)
 
 	// receiver: paych voucher list <channel> --export
-	cmd = []string{"--export", chAddr.String()}
-	list = receiverCLI.runCmd(paychVoucherListCmd, cmd)
+	list = receiverCLI.RunCmd("paych", "voucher", "list", "--export", chAddr.String())
 
 	// Check that voucher list output is correct on receiver
 	checkVoucherOutput(t, list, vouchers)
 
 	// receiver: paych voucher best-spendable <channel>
-	cmd = []string{"--export", chAddr.String()}
-	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+	bestSpendable = receiverCLI.RunCmd("paych", "voucher", "best-spendable", "--export", chAddr.String())
 
 	// Check that best spendable output is correct on receiver
 	bestVouchers = []voucherSpec{
@@ -279,12 +259,10 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	checkVoucherOutput(t, bestSpendable, bestVouchers)
 
 	// receiver: paych voucher submit <channel> <voucher>
-	cmd = []string{chAddr.String(), voucher1}
-	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "submit", chAddr.String(), voucher1)
 
 	// receiver: paych voucher best-spendable <channel>
-	cmd = []string{"--export", chAddr.String()}
-	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+	bestSpendable = receiverCLI.RunCmd("paych", "voucher", "best-spendable", "--export", chAddr.String())
 
 	// Check that best spendable output no longer includes submitted voucher
 	bestVouchers = []voucherSpec{
@@ -295,12 +273,10 @@ func TestPaymentChannelVouchers(t *testing.T) {
 	// There are three vouchers in lane 5: 50, 70, 80
 	// Submit the voucher for 50. Best spendable should still be 80.
 	// receiver: paych voucher submit <channel> <voucher>
-	cmd = []string{chAddr.String(), voucher2}
-	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "submit", chAddr.String(), voucher2)
 
 	// receiver: paych voucher best-spendable <channel>
-	cmd = []string{"--export", chAddr.String()}
-	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+	bestSpendable = receiverCLI.RunCmd("paych", "voucher", "best-spendable", "--export", chAddr.String())
 
 	// Check that best spendable output still includes the voucher for 80
 	bestVouchers = []voucherSpec{
@@ -310,12 +286,10 @@ func TestPaymentChannelVouchers(t *testing.T) {
 
 	// Submit the voucher for 80
 	// receiver: paych voucher submit <channel> <voucher>
-	cmd = []string{chAddr.String(), voucher4}
-	receiverCLI.runCmd(paychVoucherSubmitCmd, cmd)
+	receiverCLI.RunCmd("paych", "voucher", "submit", chAddr.String(), voucher4)
 
 	// receiver: paych voucher best-spendable <channel>
-	cmd = []string{"--export", chAddr.String()}
-	bestSpendable = receiverCLI.runCmd(paychVoucherBestSpendableCmd, cmd)
+	bestSpendable = receiverCLI.RunCmd("paych", "voucher", "best-spendable", "--export", chAddr.String())
 
 	// Check that best spendable output no longer includes submitted voucher
 	bestVouchers = []voucherSpec{}
@@ -326,22 +300,27 @@ func TestPaymentChannelVouchers(t *testing.T) {
 // is greater than what's left in the channel, voucher create fails
 func TestPaymentChannelVoucherCreateShortfall(t *testing.T) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+	clitest.QuietMiningLogs()
 
 	blocktime := 5 * time.Millisecond
 	ctx := context.Background()
-	nodes, addrs := startTwoNodesOneMiner(ctx, t, blocktime)
+	nodes, addrs := clitest.StartTwoNodesOneMiner(ctx, t, blocktime)
 	paymentCreator := nodes[0]
 	creatorAddr := addrs[0]
 	receiverAddr := addrs[1]
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t)
-	creatorCLI := mockCLI.client(paymentCreator.ListenAddr)
+	mockCLI := clitest.NewMockCLI(ctx, t, Commands)
+	creatorCLI := mockCLI.Client(paymentCreator.ListenAddr)
 
 	// creator: paych add-funds <creator> <receiver> <amount>
 	channelAmt := 100
-	cmd := []string{creatorAddr.String(), receiverAddr.String(), fmt.Sprintf("%d", channelAmt)}
-	chstr := creatorCLI.runCmd(paychAddFundsCmd, cmd)
+	chstr := creatorCLI.RunCmd(
+		"paych",
+		"add-funds",
+		creatorAddr.String(),
+		receiverAddr.String(),
+		fmt.Sprintf("%d", channelAmt))
 
 	chAddr, err := address.NewFromString(chstr)
 	require.NoError(t, err)
@@ -349,15 +328,25 @@ func TestPaymentChannelVoucherCreateShortfall(t *testing.T) {
 	// creator: paych voucher create <channel> <amount> --lane=1
 	voucherAmt1 := 60
 	lane1 := "--lane=1"
-	cmd = []string{lane1, chAddr.String(), strconv.Itoa(voucherAmt1)}
-	voucher1 := creatorCLI.runCmd(paychVoucherCreateCmd, cmd)
+	voucher1 := creatorCLI.RunCmd(
+		"paych",
+		"voucher",
+		"create",
+		lane1,
+		chAddr.String(),
+		strconv.Itoa(voucherAmt1))
 	fmt.Println(voucher1)
 
 	// creator: paych voucher create <channel> <amount> --lane=2
 	lane2 := "--lane=2"
 	voucherAmt2 := 70
-	cmd = []string{lane2, chAddr.String(), strconv.Itoa(voucherAmt2)}
-	_, err = creatorCLI.runCmdRaw(paychVoucherCreateCmd, cmd)
+	_, err = creatorCLI.RunCmdRaw(
+		"paych",
+		"voucher",
+		"create",
+		lane2,
+		chAddr.String(),
+		strconv.Itoa(voucherAmt2))
 
 	// Should fail because channel doesn't have required amount
 	require.Error(t, err)
@@ -388,129 +377,6 @@ func checkVoucherOutput(t *testing.T, list string, vouchers []voucherSpec) {
 	}
 }
 
-func startTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) {
-	n, sn := builder.RPCMockSbBuilder(t, test.TwoFull, test.OneMiner)
-
-	paymentCreator := n[0]
-	paymentReceiver := n[1]
-	miner := sn[0]
-
-	// Get everyone connected
-	addrs, err := paymentCreator.NetAddrsListen(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := paymentReceiver.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := miner.NetConnect(ctx, addrs); err != nil {
-		t.Fatal(err)
-	}
-
-	// Start mining blocks
-	bm := test.NewBlockMiner(ctx, t, miner, blocktime)
-	bm.MineBlocks()
-
-	// Send some funds to register the receiver
-	receiverAddr, err := paymentReceiver.WalletNew(ctx, types.KTSecp256k1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	test.SendFunds(ctx, t, paymentCreator, receiverAddr, abi.NewTokenAmount(1e18))
-
-	// Get the creator's address
-	creatorAddr, err := paymentCreator.WalletDefaultAddress(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create mock CLI
-	return n, []address.Address{creatorAddr, receiverAddr}
-}
-
-type mockCLI struct {
-	t    *testing.T
-	cctx *cli.Context
-	out  *bytes.Buffer
-}
-
-// TODO: refactor to use the methods in cli/test/mockcli.go
-func newMockCLI(t *testing.T) *mockCLI {
-	// Create a CLI App with an --api-url flag so that we can specify which node
-	// the command should be executed against
-	app := cli.NewApp()
-	app.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:   "api-url",
-			Hidden: true,
-		},
-	}
-	var out bytes.Buffer
-	app.Writer = &out
-	app.Setup()
-
-	cctx := cli.NewContext(app, &flag.FlagSet{}, nil)
-	return &mockCLI{t: t, cctx: cctx, out: &out}
-}
-
-func (c *mockCLI) client(addr multiaddr.Multiaddr) *mockCLIClient {
-	return &mockCLIClient{t: c.t, addr: addr, cctx: c.cctx, out: c.out}
-}
-
-// mockCLIClient runs commands against a particular node
-type mockCLIClient struct {
-	t    *testing.T
-	addr multiaddr.Multiaddr
-	cctx *cli.Context
-	out  *bytes.Buffer
-}
-
-func (c *mockCLIClient) runCmd(cmd *cli.Command, input []string) string {
-	out, err := c.runCmdRaw(cmd, input)
-	require.NoError(c.t, err)
-
-	return out
-}
-
-func (c *mockCLIClient) runCmdRaw(cmd *cli.Command, input []string) (string, error) {
-	// prepend --api-url=<node api listener address>
-	apiFlag := "--api-url=" + c.addr.String()
-	input = append([]string{apiFlag}, input...)
-
-	fs := c.flagSet(cmd)
-	err := fs.Parse(input)
-	require.NoError(c.t, err)
-
-	err = cmd.Action(cli.NewContext(c.cctx.App, fs, c.cctx))
-
-	// Get the output
-	str := strings.TrimSpace(c.out.String())
-	c.out.Reset()
-	return str, err
-}
-
-func (c *mockCLIClient) flagSet(cmd *cli.Command) *flag.FlagSet {
-	// Apply app level flags (so we can process --api-url flag)
-	fs := &flag.FlagSet{}
-	for _, f := range c.cctx.App.Flags {
-		err := f.Apply(fs)
-		if err != nil {
-			c.t.Fatal(err)
-		}
-	}
-	// Apply command level flags
-	for _, f := range cmd.Flags {
-		err := f.Apply(fs)
-		if err != nil {
-			c.t.Fatal(err)
-		}
-	}
-	return fs
-}
-
 // waitForHeight waits for the node to reach the given chain epoch
 func waitForHeight(ctx context.Context, t *testing.T, node test.TestNode, height abi.ChainEpoch) {
 	atHeight := make(chan struct{})

cli/state.go

@@ -15,13 +15,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 
-	"github.com/multiformats/go-multiaddr"
-
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/multiformats/go-multiaddr"
 	"github.com/multiformats/go-multihash"
 	"github.com/urfave/cli/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
@@ -107,13 +107,18 @@ var stateMinerInfo = &cli.Command{
 			return err
 		}
 
+		availableBalance, err := api.StateMinerAvailableBalance(ctx, addr, ts.Key())
+		if err != nil {
+			return xerrors.Errorf("getting miner available balance: %w", err)
+		}
+		fmt.Printf("Available Balance: %s\n", types.FIL(availableBalance))
 		fmt.Printf("Owner:\t%s\n", mi.Owner)
 		fmt.Printf("Worker:\t%s\n", mi.Worker)
 		for i, controlAddress := range mi.ControlAddresses {
 			fmt.Printf("Control %d: \t%s\n", i, controlAddress)
 		}
+
 		fmt.Printf("PeerID:\t%s\n", mi.PeerId)
-		fmt.Printf("SectorSize:\t%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize)
 		fmt.Printf("Multiaddrs: \t")
 		for _, addr := range mi.Multiaddrs {
 			a, err := multiaddr.NewMultiaddrBytes(addr)
@@ -122,6 +127,26 @@ var stateMinerInfo = &cli.Command{
 			}
 			fmt.Printf("%s ", a)
 		}
+
+		fmt.Printf("SectorSize:\t%s (%d)\n", types.SizeStr(types.NewInt(uint64(mi.SectorSize))), mi.SectorSize)
+		pow, err := api.StateMinerPower(ctx, addr, ts.Key())
+		if err != nil {
+			return err
+		}
+
+		rpercI := types.BigDiv(types.BigMul(pow.MinerPower.RawBytePower, types.NewInt(1000000)), pow.TotalPower.RawBytePower)
+		qpercI := types.BigDiv(types.BigMul(pow.MinerPower.QualityAdjPower, types.NewInt(1000000)), pow.TotalPower.QualityAdjPower)
+
+		fmt.Printf("Byte Power:   %s / %s (%0.4f%%)\n",
+			color.BlueString(types.SizeStr(pow.MinerPower.RawBytePower)),
+			types.SizeStr(pow.TotalPower.RawBytePower),
+			float64(rpercI.Int64())/10000)
+
+		fmt.Printf("Actual Power: %s / %s (%0.4f%%)\n",
+			color.GreenString(types.DeciStr(pow.MinerPower.QualityAdjPower)),
+			types.DeciStr(pow.TotalPower.QualityAdjPower),
+			float64(qpercI.Int64())/10000)
+
 		fmt.Println()
 
 		cd, err := api.StateMinerProvingDeadline(ctx, addr, ts.Key())
@@ -1299,12 +1324,11 @@ func sumGas(changes []*types.GasTrace) types.GasTrace {
 }
 
 func JsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) {
-	methodMeta, found := stmgr.MethodsMap[code][method]
-	if !found {
-		return "", fmt.Errorf("method %d not found on actor %s", method, code)
+	p, err := stmgr.GetParamType(code, method)
+	if err != nil {
+		return "", err
 	}
-	re := reflect.New(methodMeta.Params.Elem())
-	p := re.Interface().(cbg.CBORUnmarshaler)
+
 	if err := p.UnmarshalCBOR(bytes.NewReader(params)); err != nil {
 		return "", err
 	}

cli/sync.go

@@ -45,8 +45,8 @@ var syncStatusCmd = &cli.Command{
 		}
 
 		fmt.Println("sync status:")
-		for i, ss := range state.ActiveSyncs {
-			fmt.Printf("worker %d:\n", i)
+		for _, ss := range state.ActiveSyncs {
+			fmt.Printf("worker %d:\n", ss.WorkerID)
 			var base, target []cid.Cid
 			var heightDiff int64
 			var theight abi.ChainEpoch
@@ -263,12 +263,17 @@ func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error {
 			return err
 		}
 
+		if len(state.ActiveSyncs) == 0 {
+			time.Sleep(time.Second)
+			continue
+		}
+
 		head, err := napi.ChainHead(ctx)
 		if err != nil {
 			return err
 		}
 
-		working := 0
+		working := -1
 		for i, ss := range state.ActiveSyncs {
 			switch ss.Stage {
 			case api.StageSyncComplete:
@@ -279,7 +284,12 @@ func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error {
 			}
 		}
 
+		if working == -1 {
+			working = len(state.ActiveSyncs) - 1
+		}
+
 		ss := state.ActiveSyncs[working]
+		workerID := ss.WorkerID
 
 		var baseHeight abi.ChainEpoch
 		var target []cid.Cid
@@ -302,7 +312,7 @@ func SyncWait(ctx context.Context, napi api.FullNode, watch bool) error {
 			fmt.Print("\r\x1b[2K\x1b[A")
 		}
 
-		fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", working, baseHeight, theight, heightDiff)
+		fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", workerID, baseHeight, theight, heightDiff)
 		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d\n", ss.Stage, ss.Height, theight-ss.Height)
 		lastLines = 2
 

cli/test/client.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/lotus/api/test"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -25,8 +27,8 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 	defer cancel()
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t, cmds)
-	clientCLI := mockCLI.client(clientNode.ListenAddr)
+	mockCLI := NewMockCLI(ctx, t, cmds)
+	clientCLI := mockCLI.Client(clientNode.ListenAddr)
 
 	// Get the miner address
 	addrs, err := clientNode.StateListMiners(ctx, types.EmptyTSK)
@@ -37,10 +39,7 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 	fmt.Println("Miner:", minerAddr)
 
 	// client query-ask <miner addr>
-	cmd := []string{
-		"client", "query-ask", minerAddr.String(),
-	}
-	out := clientCLI.runCmd(cmd)
+	out := clientCLI.RunCmd("client", "query-ask", minerAddr.String())
 	require.Regexp(t, regexp.MustCompile("Ask:"), out)
 
 	// Create a deal (non-interactive)
@@ -50,10 +49,7 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 	dataCid := res.Root
 	price := "1000000attofil"
 	duration := fmt.Sprintf("%d", build.MinDealDuration)
-	cmd = []string{
-		"client", "deal", dataCid.String(), minerAddr.String(), price, duration,
-	}
-	out = clientCLI.runCmd(cmd)
+	out = clientCLI.RunCmd("client", "deal", dataCid.String(), minerAddr.String(), price, duration)
 	fmt.Println("client deal", out)
 
 	// Create a deal (interactive)
@@ -67,9 +63,7 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 	require.NoError(t, err)
 	dataCid2 := res.Root
 	duration = fmt.Sprintf("%d", build.MinDealDuration/builtin.EpochsInDay)
-	cmd = []string{
-		"client", "deal",
-	}
+	cmd := []string{"client", "deal"}
 	interactiveCmds := []string{
 		dataCid2.String(),
 		duration,
@@ -77,15 +71,14 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 		"no",
 		"yes",
 	}
-	out = clientCLI.runInteractiveCmd(cmd, interactiveCmds)
+	out = clientCLI.RunInteractiveCmd(cmd, interactiveCmds)
 	fmt.Println("client deal:\n", out)
 
 	// Wait for provider to start sealing deal
 	dealStatus := ""
-	for dealStatus != "StorageDealSealing" {
+	for {
 		// client list-deals
-		cmd = []string{"client", "list-deals"}
-		out = clientCLI.runCmd(cmd)
+		out = clientCLI.RunCmd("client", "list-deals")
 		fmt.Println("list-deals:\n", out)
 
 		lines := strings.Split(out, "\n")
@@ -97,6 +90,9 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 		}
 		dealStatus = parts[3]
 		fmt.Println("  Deal status:", dealStatus)
+		if dealComplete(t, dealStatus) {
+			break
+		}
 
 		time.Sleep(time.Second)
 	}
@@ -106,10 +102,18 @@ func RunClientTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNode)
 	tmpdir, err := ioutil.TempDir(os.TempDir(), "test-cli-client")
 	require.NoError(t, err)
 	path := filepath.Join(tmpdir, "outfile.dat")
-	cmd = []string{
-		"client", "retrieve", dataCid.String(), path,
-	}
-	out = clientCLI.runCmd(cmd)
+	out = clientCLI.RunCmd("client", "retrieve", dataCid.String(), path)
 	fmt.Println("retrieve:\n", out)
 	require.Regexp(t, regexp.MustCompile("Success"), out)
 }
+
+func dealComplete(t *testing.T, dealStatus string) bool {
+	switch dealStatus {
+	case "StorageDealFailing", "StorageDealError":
+		t.Fatal(xerrors.Errorf("Storage deal failed with status: " + dealStatus))
+	case "StorageDealStaged", "StorageDealSealing", "StorageDealActive", "StorageDealExpired", "StorageDealSlashed":
+		return true
+	}
+
+	return false
+}

cli/test/mockcli.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"bytes"
+	"context"
 	"flag"
 	"strings"
 	"testing"
@@ -11,14 +12,14 @@ import (
 	lcli "github.com/urfave/cli/v2"
 )
 
-type mockCLI struct {
+type MockCLI struct {
 	t    *testing.T
 	cmds []*lcli.Command
 	cctx *lcli.Context
 	out  *bytes.Buffer
 }
 
-func newMockCLI(t *testing.T, cmds []*lcli.Command) *mockCLI {
+func NewMockCLI(ctx context.Context, t *testing.T, cmds []*lcli.Command) *MockCLI {
 	// Create a CLI App with an --api-url flag so that we can specify which node
 	// the command should be executed against
 	app := &lcli.App{
@@ -36,15 +37,16 @@ func newMockCLI(t *testing.T, cmds []*lcli.Command) *mockCLI {
 	app.Setup()
 
 	cctx := lcli.NewContext(app, &flag.FlagSet{}, nil)
-	return &mockCLI{t: t, cmds: cmds, cctx: cctx, out: &out}
+	cctx.Context = ctx
+	return &MockCLI{t: t, cmds: cmds, cctx: cctx, out: &out}
 }
 
-func (c *mockCLI) client(addr multiaddr.Multiaddr) *mockCLIClient {
-	return &mockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out}
+func (c *MockCLI) Client(addr multiaddr.Multiaddr) *MockCLIClient {
+	return &MockCLIClient{t: c.t, cmds: c.cmds, addr: addr, cctx: c.cctx, out: c.out}
 }
 
-// mockCLIClient runs commands against a particular node
-type mockCLIClient struct {
+// MockCLIClient runs commands against a particular node
+type MockCLIClient struct {
 	t    *testing.T
 	cmds []*lcli.Command
 	addr multiaddr.Multiaddr
@@ -52,42 +54,48 @@ type mockCLIClient struct {
 	out  *bytes.Buffer
 }
 
-func (c *mockCLIClient) run(cmd []string, params []string, args []string) string {
-	// Add parameter --api-url=<node api listener address>
-	apiFlag := "--api-url=" + c.addr.String()
-	params = append([]string{apiFlag}, params...)
-
-	err := c.cctx.App.Run(append(append(cmd, params...), args...))
-	require.NoError(c.t, err)
-
-	// Get the output
-	str := strings.TrimSpace(c.out.String())
-	c.out.Reset()
-	return str
-}
-
-func (c *mockCLIClient) runCmd(input []string) string {
-	cmd := c.cmdByNameSub(input[0], input[1])
-	out, err := c.runCmdRaw(cmd, input[2:])
+func (c *MockCLIClient) RunCmd(input ...string) string {
+	out, err := c.RunCmdRaw(input...)
 	require.NoError(c.t, err)
 
 	return out
 }
 
-func (c *mockCLIClient) cmdByNameSub(name string, sub string) *lcli.Command {
-	for _, c := range c.cmds {
-		if c.Name == name {
-			for _, s := range c.Subcommands {
-				if s.Name == sub {
-					return s
-				}
-			}
+// Given an input, find the corresponding command or sub-command.
+// eg "paych add-funds"
+func (c *MockCLIClient) cmdByNameSub(input []string) (*lcli.Command, []string) {
+	name := input[0]
+	for _, cmd := range c.cmds {
+		if cmd.Name == name {
+			return c.findSubcommand(cmd, input[1:])
 		}
 	}
-	return nil
+	return nil, []string{}
 }
 
-func (c *mockCLIClient) runCmdRaw(cmd *lcli.Command, input []string) (string, error) {
+func (c *MockCLIClient) findSubcommand(cmd *lcli.Command, input []string) (*lcli.Command, []string) {
+	// If there are no sub-commands, return the current command
+	if len(cmd.Subcommands) == 0 {
+		return cmd, input
+	}
+
+	// Check each sub-command for a match against the name
+	subName := input[0]
+	for _, subCmd := range cmd.Subcommands {
+		if subCmd.Name == subName {
+			// Found a match, recursively search for sub-commands
+			return c.findSubcommand(subCmd, input[1:])
+		}
+	}
+	return nil, []string{}
+}
+
+func (c *MockCLIClient) RunCmdRaw(input ...string) (string, error) {
+	cmd, input := c.cmdByNameSub(input)
+	if cmd == nil {
+		panic("Could not find command " + input[0] + " " + input[1])
+	}
+
 	// prepend --api-url=<node api listener address>
 	apiFlag := "--api-url=" + c.addr.String()
 	input = append([]string{apiFlag}, input...)
@@ -104,7 +112,7 @@ func (c *mockCLIClient) runCmdRaw(cmd *lcli.Command, input []string) (string, er
 	return str, err
 }
 
-func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet {
+func (c *MockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet {
 	// Apply app level flags (so we can process --api-url flag)
 	fs := &flag.FlagSet{}
 	for _, f := range c.cctx.App.Flags {
@@ -123,11 +131,11 @@ func (c *mockCLIClient) flagSet(cmd *lcli.Command) *flag.FlagSet {
 	return fs
 }
 
-func (c *mockCLIClient) runInteractiveCmd(cmd []string, interactive []string) string {
+func (c *MockCLIClient) RunInteractiveCmd(cmd []string, interactive []string) string {
 	c.toStdin(strings.Join(interactive, "\n") + "\n")
-	return c.runCmd(cmd)
+	return c.RunCmd(cmd...)
 }
 
-func (c *mockCLIClient) toStdin(s string) {
+func (c *MockCLIClient) toStdin(s string) {
 	c.cctx.App.Metadata["stdin"] = bytes.NewBufferString(s)
 }

cli/test/multisig.go

@@ -18,8 +18,8 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 	ctx := context.Background()
 
 	// Create mock CLI
-	mockCLI := newMockCLI(t, cmds)
-	clientCLI := mockCLI.client(clientNode.ListenAddr)
+	mockCLI := NewMockCLI(ctx, t, cmds)
+	clientCLI := mockCLI.Client(clientNode.ListenAddr)
 
 	// Create some wallets on the node to use for testing multisig
 	var walletAddrs []address.Address
@@ -39,7 +39,7 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 	paramDuration := "--duration=50"
 	paramRequired := fmt.Sprintf("--required=%d", threshold)
 	paramValue := fmt.Sprintf("--value=%dattofil", amtAtto)
-	cmd := []string{
+	out := clientCLI.RunCmd(
 		"msig", "create",
 		paramRequired,
 		paramDuration,
@@ -47,8 +47,7 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 		walletAddrs[0].String(),
 		walletAddrs[1].String(),
 		walletAddrs[2].String(),
-	}
-	out := clientCLI.runCmd(cmd)
+	)
 	fmt.Println(out)
 
 	// Extract msig robust address from output
@@ -62,18 +61,16 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 	// Propose to add a new address to the msig
 	// msig add-propose --from=<addr> <msig> <addr>
 	paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0])
-	cmd = []string{
+	out = clientCLI.RunCmd(
 		"msig", "add-propose",
 		paramFrom,
 		msigRobustAddr,
 		walletAddrs[3].String(),
-	}
-	out = clientCLI.runCmd(cmd)
+	)
 	fmt.Println(out)
 
 	// msig inspect <msig>
-	cmd = []string{"msig", "inspect", "--vesting", "--decode-params", msigRobustAddr}
-	out = clientCLI.runCmd(cmd)
+	out = clientCLI.RunCmd("msig", "inspect", "--vesting", "--decode-params", msigRobustAddr)
 	fmt.Println(out)
 
 	// Expect correct balance
@@ -87,7 +84,7 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 	// msig add-approve --from=<addr> <msig> <addr> 0 <addr> false
 	txnID := "0"
 	paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1])
-	cmd = []string{
+	out = clientCLI.RunCmd(
 		"msig", "add-approve",
 		paramFrom,
 		msigRobustAddr,
@@ -95,7 +92,6 @@ func RunMultisigTest(t *testing.T, cmds []*lcli.Command, clientNode test.TestNod
 		txnID,
 		walletAddrs[3].String(),
 		"false",
-	}
-	out = clientCLI.runCmd(cmd)
+	)
 	fmt.Println(out)
 }

cli/test/net.go

@@ -5,6 +5,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/types"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api/test"
 	test2 "github.com/filecoin-project/lotus/node/test"
@@ -39,3 +42,46 @@ func StartOneNodeOneMiner(ctx context.Context, t *testing.T, blocktime time.Dura
 	// Create mock CLI
 	return full, fullAddr
 }
+
+func StartTwoNodesOneMiner(ctx context.Context, t *testing.T, blocktime time.Duration) ([]test.TestNode, []address.Address) {
+	n, sn := test2.RPCMockSbBuilder(t, test.TwoFull, test.OneMiner)
+
+	fullNode1 := n[0]
+	fullNode2 := n[1]
+	miner := sn[0]
+
+	// Get everyone connected
+	addrs, err := fullNode1.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := fullNode2.NetConnect(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Start mining blocks
+	bm := test.NewBlockMiner(ctx, t, miner, blocktime)
+	bm.MineBlocks()
+
+	// Send some funds to register the second node
+	fullNodeAddr2, err := fullNode2.WalletNew(ctx, types.KTSecp256k1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	test.SendFunds(ctx, t, fullNode1, fullNodeAddr2, abi.NewTokenAmount(1e18))
+
+	// Get the first node's address
+	fullNodeAddr1, err := fullNode1.WalletDefaultAddress(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create mock CLI
+	return n, []address.Address{fullNodeAddr1, fullNodeAddr2}
+}

cmd/lotus-bench/import.go

@@ -16,21 +16,30 @@ import (
 	"sort"
 	"time"
 
+	ocprom "contrib.go.opencensus.io/exporter/prometheus"
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
+	"github.com/ipfs/go-cid"
+	metricsi "github.com/ipfs/go-metrics-interface"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
-	metricsprometheus "github.com/ipfs/go-metrics-prometheus"
-	"github.com/ipld/go-car"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/filecoin-project/lotus/node/repo"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	metricsprometheus "github.com/ipfs/go-metrics-prometheus"
+	"github.com/ipld/go-car"
+
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 
 	bdg "github.com/dgraph-io/badger/v2"
@@ -56,9 +65,25 @@ var importBenchCmd = &cli.Command{
 		importAnalyzeCmd,
 	},
 	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "start-tipset",
+			Usage: "start validation at the given tipset key; in format cid1,cid2,cid3...",
+		},
+		&cli.StringFlag{
+			Name:  "end-tipset",
+			Usage: "halt validation at the given tipset key; in format cid1,cid2,cid3...",
+		},
+		&cli.StringFlag{
+			Name:  "genesis-tipset",
+			Usage: "genesis tipset key; in format cid1,cid2,cid3...",
+		},
 		&cli.Int64Flag{
-			Name:  "height",
-			Usage: "halt validation after given height",
+			Name:  "start-height",
+			Usage: "start validation at given height; beware that chain traversal by height is very slow",
+		},
+		&cli.Int64Flag{
+			Name:  "end-height",
+			Usage: "halt validation after given height; beware that chain traversal by height is very slow",
 		},
 		&cli.IntFlag{
 			Name:  "batch-seal-verify-threads",
@@ -86,32 +111,52 @@ var importBenchCmd = &cli.Command{
 			Name:  "global-profile",
 			Value: true,
 		},
-		&cli.Int64Flag{
-			Name: "start-at",
-		},
 		&cli.BoolFlag{
 			Name: "only-import",
 		},
 		&cli.BoolFlag{
 			Name: "use-pebble",
 		},
+		&cli.BoolFlag{
+			Name: "use-native-badger",
+		},
+		&cli.StringFlag{
+			Name: "car",
+			Usage: "path to CAR file; required for import; on validation, either " +
+				"a CAR path or the --head flag are required",
+		},
+		&cli.StringFlag{
+			Name: "head",
+			Usage: "tipset key of the head, useful when benchmarking validation " +
+				"on an existing chain store, where a CAR is not available; " +
+				"if both --car and --head are provided, --head takes precedence " +
+				"over the CAR root; the format is cid1,cid2,cid3...",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		metricsprometheus.Inject() //nolint:errcheck
 		vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads")
-		if !cctx.Args().Present() {
-			fmt.Println("must pass car file of chain to benchmark importing")
-			return nil
-		}
-
-		cfi, err := os.Open(cctx.Args().First())
-		if err != nil {
-			return err
-		}
-		defer cfi.Close() //nolint:errcheck // read only file
 
 		go func() {
-			http.Handle("/debug/metrics/prometheus", promhttp.Handler())
+			// Prometheus globals are exposed as interfaces, but the prometheus
+			// OpenCensus exporter expects a concrete *Registry. The concrete type of
+			// the globals are actually *Registry, so we downcast them, staying
+			// defensive in case things change under the hood.
+			registry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry)
+			if !ok {
+				log.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", prometheus.DefaultRegisterer)
+				return
+			}
+			exporter, err := ocprom.NewExporter(ocprom.Options{
+				Registry:  registry,
+				Namespace: "lotus",
+			})
+			if err != nil {
+				log.Fatalf("could not create the prometheus stats exporter: %v", err)
+			}
+
+			http.Handle("/debug/metrics", exporter)
+
 			http.ListenAndServe("localhost:6060", nil) //nolint:errcheck
 		}()
 
@@ -126,17 +171,17 @@ var importBenchCmd = &cli.Command{
 			tdir = tmp
 		}
 
-		bdgOpt := badger.DefaultOptions
-		bdgOpt.GcInterval = 0
-		bdgOpt.Options = bdg.DefaultOptions("")
-		bdgOpt.Options.SyncWrites = false
-		bdgOpt.Options.Truncate = true
-		bdgOpt.Options.DetectConflicts = false
+		var (
+			ds  datastore.Batching
+			bs  blockstore.Blockstore
+			err error
+		)
 
-		var bds datastore.Batching
-		if cctx.Bool("use-pebble") {
+		switch {
+		case cctx.Bool("use-pebble"):
+			log.Info("using pebble")
 			cache := 512
-			bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
+			ds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
 				// Pebble has a single combined cache area and the write
 				// buffers are taken from this too. Assign all available
 				// memory allowance for cache.
@@ -155,30 +200,53 @@ var importBenchCmd = &cli.Command{
 				},
 				Logger: log,
 			})
-		} else {
-			bds, err = badger.NewDatastore(tdir, &bdgOpt)
+
+		case cctx.Bool("use-native-badger"):
+			log.Info("using native badger")
+			var opts badgerbs.Options
+			if opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, tdir, false); err != nil {
+				return err
+			}
+			opts.SyncWrites = false
+			bs, err = badgerbs.Open(opts)
+
+		default: // legacy badger via datastore.
+			log.Info("using legacy badger")
+			bdgOpt := badger.DefaultOptions
+			bdgOpt.GcInterval = 0
+			bdgOpt.Options = bdg.DefaultOptions("")
+			bdgOpt.Options.SyncWrites = false
+			bdgOpt.Options.Truncate = true
+			bdgOpt.Options.DetectConflicts = false
+
+			ds, err = badger.NewDatastore(tdir, &bdgOpt)
 		}
+
 		if err != nil {
 			return err
 		}
-		defer bds.Close() //nolint:errcheck
 
-		bds = measure.New("dsbench", bds)
+		if ds != nil {
+			ds = measure.New("dsbench", ds)
+			defer ds.Close() //nolint:errcheck
+			bs = blockstore.NewBlockstore(ds)
+		}
 
-		bs := blockstore.NewBlockstore(bds)
+		if c, ok := bs.(io.Closer); ok {
+			defer c.Close() //nolint:errcheck
+		}
+
+		ctx := metricsi.CtxScope(context.Background(), "lotus")
 		cacheOpts := blockstore.DefaultCacheOpts()
 		cacheOpts.HasBloomFilterSize = 0
-
-		cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts)
+		bs, err = blockstore.CachedBlockstore(ctx, bs, cacheOpts)
 		if err != nil {
 			return err
 		}
-		bs = cbs
-		ds := datastore.NewMapDatastore()
 
 		var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
 		if cctx.IsSet("syscall-cache") {
-			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt)
+			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions)
 			if err != nil {
 				return xerrors.Errorf("opening syscall-cache datastore: %w", err)
 			}
@@ -193,11 +261,221 @@ var importBenchCmd = &cli.Command{
 			return nil
 		}
 
-		cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil)
+		metadataDs := datastore.NewMapDatastore()
+		cs := store.NewChainStore(bs, bs, metadataDs, vm.Syscalls(verifier), nil)
 		stm := stmgr.NewStateManager(cs)
 
+		startTime := time.Now()
+
+		// register a gauge that reports how long since the measurable
+		// operation began.
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "lotus_bench_time_taken_secs",
+		}, func() float64 {
+			return time.Since(startTime).Seconds()
+		})
+
+		defer func() {
+			end := time.Now().Format(time.RFC3339)
+
+			resp, err := http.Get("http://localhost:6060/debug/metrics")
+			if err != nil {
+				log.Warnf("failed to scape prometheus: %s", err)
+			}
+
+			metricsfi, err := os.Create("bench.metrics")
+			if err != nil {
+				log.Warnf("failed to write prometheus data: %s", err)
+			}
+
+			_, _ = io.Copy(metricsfi, resp.Body) //nolint:errcheck
+			_ = metricsfi.Close()                //nolint:errcheck
+
+			writeProfile := func(name string) {
+				if file, err := os.Create(fmt.Sprintf("%s.%s.%s.pprof", name, startTime.Format(time.RFC3339), end)); err == nil {
+					if err := pprof.Lookup(name).WriteTo(file, 0); err != nil {
+						log.Warnf("failed to write %s pprof: %s", name, err)
+					}
+					_ = file.Close()
+				} else {
+					log.Warnf("failed to create %s pprof file: %s", name, err)
+				}
+			}
+
+			writeProfile("heap")
+			writeProfile("allocs")
+		}()
+
+		var carFile *os.File
+
+		// open the CAR file if one is provided.
+		if path := cctx.String("car"); path != "" {
+			var err error
+			if carFile, err = os.Open(path); err != nil {
+				return xerrors.Errorf("failed to open provided CAR file: %w", err)
+			}
+		}
+
+		var head *types.TipSet
+
+		// --- IMPORT ---
+		if !cctx.Bool("no-import") {
+			if cctx.Bool("global-profile") {
+				prof, err := os.Create("bench.import.pprof")
+				if err != nil {
+					return err
+				}
+				defer prof.Close() //nolint:errcheck
+
+				if err := pprof.StartCPUProfile(prof); err != nil {
+					return err
+				}
+			}
+
+			// import is NOT suppressed; do it.
+			if carFile == nil { // a CAR is compulsory for the import.
+				return fmt.Errorf("no CAR file provided for import")
+			}
+
+			head, err = cs.Import(carFile)
+			if err != nil {
+				return err
+			}
+
+			pprof.StopCPUProfile()
+		}
+
+		if cctx.Bool("only-import") {
+			return nil
+		}
+
+		// --- VALIDATION ---
+		//
+		// we are now preparing for the validation benchmark.
+		// a HEAD needs to be set; --head takes precedence over the root
+		// of the CAR, if both are provided.
+		if h := cctx.String("head"); h != "" {
+			cids, err := lcli.ParseTipSetString(h)
+			if err != nil {
+				return xerrors.Errorf("failed to parse head tipset key: %w", err)
+			}
+
+			head, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+			if err != nil {
+				return err
+			}
+		} else if carFile != nil && head == nil {
+			cr, err := car.NewCarReader(carFile)
+			if err != nil {
+				return err
+			}
+			head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
+			if err != nil {
+				return err
+			}
+		} else if h == "" && carFile == nil {
+			return xerrors.Errorf("neither --car nor --head flags supplied")
+		}
+
+		log.Infof("chain head is tipset: %s", head.Key())
+
+		var genesis *types.TipSet
+		log.Infof("getting genesis block")
+		if tsk := cctx.String("genesis-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to parse genesis tipset key: %w", err)
+			}
+			genesis, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else {
+			log.Warnf("getting genesis by height; this will be slow; pass in the genesis tipset through --genesis-tipset")
+			// fallback to the slow path of walking the chain.
+			genesis, err = cs.GetTipsetByHeight(context.TODO(), 0, head, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if err = cs.SetGenesis(genesis.Blocks()[0]); err != nil {
+			return err
+		}
+
+		// Resolve the end tipset, falling back to head if not provided.
+		end := head
+		if tsk := cctx.String("end-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to end genesis tipset key: %w", err)
+			}
+			end, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else if h := cctx.Int64("end-height"); h != 0 {
+			log.Infof("getting end tipset at height %d...", h)
+			end, err = cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		// Resolve the start tipset, if provided; otherwise, fallback to
+		// height 1 for a start point.
+		var (
+			startEpoch = abi.ChainEpoch(1)
+			start      *types.TipSet
+		)
+
+		if tsk := cctx.String("start-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to start genesis tipset key: %w", err)
+			}
+			start, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else if h := cctx.Int64("start-height"); h != 0 {
+			log.Infof("getting start tipset at height %d...", h)
+			// lookback from the end tipset (which falls back to head if not supplied).
+			start, err = cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), end, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if start != nil {
+			startEpoch = start.Height()
+			if err := cs.ForceHeadSilent(context.Background(), start); err != nil {
+				// if err := cs.SetHead(start); err != nil {
+				return err
+			}
+		}
+
+		inverseChain := append(make([]*types.TipSet, 0, end.Height()), end)
+		for ts := end; ts.Height() > startEpoch; {
+			if h := ts.Height(); h%100 == 0 {
+				log.Infof("walking back the chain; loaded tipset at height %d...", h)
+			}
+			next, err := cs.LoadTipSet(ts.Parents())
+			if err != nil {
+				return err
+			}
+
+			inverseChain = append(inverseChain, next)
+			ts = next
+		}
+
+		var enc *json.Encoder
+		if cctx.Bool("export-traces") {
+			ibj, err := os.Create("bench.json")
+			if err != nil {
+				return err
+			}
+			defer ibj.Close() //nolint:errcheck
+
+			enc = json.NewEncoder(ibj)
+		}
+
 		if cctx.Bool("global-profile") {
-			prof, err := os.Create("import-bench.prof")
+			prof, err := os.Create("bench.validation.pprof")
 			if err != nil {
 				return err
 			}
@@ -208,84 +486,8 @@ var importBenchCmd = &cli.Command{
 			}
 		}
 
-		var head *types.TipSet
-		if !cctx.Bool("no-import") {
-			head, err = cs.Import(cfi)
-			if err != nil {
-				return err
-			}
-		} else {
-			cr, err := car.NewCarReader(cfi)
-			if err != nil {
-				return err
-			}
-			head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
-			if err != nil {
-				return err
-			}
-		}
-
-		if cctx.Bool("only-import") {
-			return nil
-		}
-
-		gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true)
-		if err != nil {
-			return err
-		}
-
-		err = cs.SetGenesis(gb.Blocks()[0])
-		if err != nil {
-			return err
-		}
-
-		startEpoch := abi.ChainEpoch(1)
-		if cctx.IsSet("start-at") {
-			startEpoch = abi.ChainEpoch(cctx.Int64("start-at"))
-			start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true)
-			if err != nil {
-				return err
-			}
-
-			err = cs.SetHead(start)
-			if err != nil {
-				return err
-			}
-		}
-
-		if h := cctx.Int64("height"); h != 0 {
-			tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
-			if err != nil {
-				return err
-			}
-			head = tsh
-		}
-
-		ts := head
-		tschain := []*types.TipSet{ts}
-		for ts.Height() > startEpoch {
-			next, err := cs.LoadTipSet(ts.Parents())
-			if err != nil {
-				return err
-			}
-
-			tschain = append(tschain, next)
-			ts = next
-		}
-
-		var enc *json.Encoder
-		if cctx.Bool("export-traces") {
-			ibj, err := os.Create("import-bench.json")
-			if err != nil {
-				return err
-			}
-			defer ibj.Close() //nolint:errcheck
-
-			enc = json.NewEncoder(ibj)
-		}
-
-		for i := len(tschain) - 1; i >= 1; i-- {
-			cur := tschain[i]
+		for i := len(inverseChain) - 1; i >= 1; i-- {
+			cur := inverseChain[i]
 			start := time.Now()
 			log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids())
 			st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
@@ -304,7 +506,7 @@ var importBenchCmd = &cli.Command{
 					return xerrors.Errorf("failed to write out tipsetexec: %w", err)
 				}
 			}
-			if tschain[i-1].ParentState() != st {
+			if inverseChain[i-1].ParentState() != st {
 				stripCallers(tse.Trace)
 				lastTrace := tse.Trace
 				d, err := json.MarshalIndent(lastTrace, "", "  ")
@@ -320,23 +522,7 @@ var importBenchCmd = &cli.Command{
 
 		pprof.StopCPUProfile()
 
-		if true {
-			resp, err := http.Get("http://localhost:6060/debug/metrics/prometheus")
-			if err != nil {
-				return err
-			}
-
-			metricsfi, err := os.Create("import-bench.metrics")
-			if err != nil {
-				return err
-			}
-
-			io.Copy(metricsfi, resp.Body) //nolint:errcheck
-			metricsfi.Close()             //nolint:errcheck
-		}
-
 		return nil
-
 	},
 }
 

cmd/lotus-bench/main.go

@@ -26,7 +26,7 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs"
-	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	lapi "github.com/filecoin-project/lotus/api"
@@ -614,7 +614,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 					if !skipunseal {
 						log.Infof("[%d] Unsealing sector", i)
 						{
-							p, done, err := sbfs.AcquireSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, stores.FTUnsealed, stores.FTNone, stores.PathSealing)
+							p, done, err := sbfs.AcquireSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
 							if err != nil {
 								return xerrors.Errorf("acquire unsealed sector for removing: %w", err)
 							}

cmd/lotus-seal-worker/cli.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var setCmd = &cli.Command{
+	Name:  "set",
+	Usage: "Manage worker settings",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "enabled",
+			Usage: "enable/disable new task processing",
+			Value: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if err := api.SetEnabled(ctx, cctx.Bool("enabled")); err != nil {
+			return xerrors.Errorf("SetEnabled: %w", err)
+		}
+
+		return nil
+	},
+}
+
+var waitQuietCmd = &cli.Command{
+	Name:  "wait-quiet",
+	Usage: "Block until all running tasks exit",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return api.WaitQuiet(ctx)
+	},
+}

cmd/lotus-seal-worker/info.go

@@ -32,6 +32,18 @@ var infoCmd = &cli.Command{
 		cli.VersionPrinter(cctx)
 		fmt.Println()
 
+		sess, err := api.ProcessSession(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting session: %w", err)
+		}
+		fmt.Printf("Session: %s\n", sess)
+
+		enabled, err := api.Enabled(ctx)
+		if err != nil {
+			return xerrors.Errorf("checking worker status: %w", err)
+		}
+		fmt.Printf("Enabled: %t", enabled)
+
 		info, err := api.Info(ctx)
 		if err != nil {
 			return xerrors.Errorf("getting info: %w", err)

cmd/lotus-seal-worker/main.go

@@ -10,11 +10,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
+	"github.com/ipfs/go-datastore/namespace"
 	logging "github.com/ipfs/go-log/v2"
 	manet "github.com/multiformats/go-multiaddr/net"
 	"github.com/urfave/cli/v2"
@@ -25,6 +25,7 @@ import (
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	paramfetch "github.com/filecoin-project/go-paramfetch"
+	"github.com/filecoin-project/go-statestore"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
@@ -37,6 +38,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/lotuslog"
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 	"github.com/filecoin-project/lotus/metrics"
+	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -56,6 +58,8 @@ func main() {
 		runCmd,
 		infoCmd,
 		storageCmd,
+		setCmd,
+		waitQuietCmd,
 	}
 
 	app := &cli.App{
@@ -173,15 +177,18 @@ var runCmd = &cli.Command{
 		}
 
 		// Connect to storage-miner
+		ctx := lcli.ReqContext(cctx)
+
 		var nodeApi api.StorageMiner
 		var closer func()
 		var err error
 		for {
-			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx,
-				jsonrpc.WithNoReconnect(),
-				jsonrpc.WithTimeout(30*time.Second))
+			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp)
 			if err == nil {
-				break
+				_, err = nodeApi.Version(ctx)
+				if err == nil {
+					break
+				}
 			}
 			fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err)
 			time.Sleep(time.Second)
@@ -189,7 +196,6 @@ var runCmd = &cli.Command{
 		}
 
 		defer closer()
-		ctx := lcli.ReqContext(cctx)
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
@@ -209,8 +215,6 @@ var runCmd = &cli.Command{
 		}
 		log.Infof("Remote version %s", v)
 
-		watchMinerConn(ctx, cctx, nodeApi)
-
 		// Check params
 
 		act, err := nodeApi.ActorAddress(ctx)
@@ -318,6 +322,15 @@ var runCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		defer func() {
+			if err := lr.Close(); err != nil {
+				log.Error("closing repo", err)
+			}
+		}()
+		ds, err := lr.Datastore("/metadata")
+		if err != nil {
+			return err
+		}
 
 		log.Info("Opening local storage; connecting to master")
 		const unspecifiedAddress = "0.0.0.0"
@@ -357,12 +370,14 @@ var runCmd = &cli.Command{
 
 		// Create / expose the worker
 
+		wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix))
+
 		workerApi := &worker{
 			LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{
 				SealProof: spt,
 				TaskTypes: taskTypes,
 				NoSwap:    cctx.Bool("no-swap"),
-			}, remote, localStore, nodeApi),
+			}, remote, localStore, nodeApi, nodeApi, wsts),
 			localStore: localStore,
 			ls:         lr,
 		}
@@ -433,13 +448,80 @@ var runCmd = &cli.Command{
 			}
 		}
 
-		log.Info("Waiting for tasks")
+		minerSession, err := nodeApi.Session(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting miner session: %w", err)
+		}
+
+		waitQuietCh := func() chan struct{} {
+			out := make(chan struct{})
+			go func() {
+				workerApi.LocalWorker.WaitQuiet()
+				close(out)
+			}()
+			return out
+		}
 
 		go func() {
-			if err := nodeApi.WorkerConnect(ctx, "ws://"+address+"/rpc/v0"); err != nil {
-				log.Errorf("Registering worker failed: %+v", err)
-				cancel()
-				return
+			heartbeats := time.NewTicker(stores.HeartbeatInterval)
+			defer heartbeats.Stop()
+
+			var redeclareStorage bool
+			var readyCh chan struct{}
+			for {
+				// If we're reconnecting, redeclare storage first
+				if redeclareStorage {
+					log.Info("Redeclaring local storage")
+
+					if err := localStore.Redeclare(ctx); err != nil {
+						log.Errorf("Redeclaring local storage failed: %+v", err)
+
+						select {
+						case <-ctx.Done():
+							return // graceful shutdown
+						case <-heartbeats.C:
+						}
+						continue
+					}
+				}
+
+				// TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly
+				if readyCh == nil {
+					log.Info("Making sure no local tasks are running")
+					readyCh = waitQuietCh()
+				}
+
+				for {
+					curSession, err := nodeApi.Session(ctx)
+					if err != nil {
+						log.Errorf("heartbeat: checking remote session failed: %+v", err)
+					} else {
+						if curSession != minerSession {
+							minerSession = curSession
+							break
+						}
+					}
+
+					select {
+					case <-readyCh:
+						if err := nodeApi.WorkerConnect(ctx, "http://"+address+"/rpc/v0"); err != nil {
+							log.Errorf("Registering worker failed: %+v", err)
+							cancel()
+							return
+						}
+
+						log.Info("Worker registered successfully, waiting for tasks")
+
+						readyCh = nil
+					case <-heartbeats.C:
+					case <-ctx.Done():
+						return // graceful shutdown
+					}
+				}
+
+				log.Errorf("LOTUS-MINER CONNECTION LOST")
+
+				redeclareStorage = true
 			}
 		}()
 
@@ -447,55 +529,6 @@ var runCmd = &cli.Command{
 	},
 }
 
-func watchMinerConn(ctx context.Context, cctx *cli.Context, nodeApi api.StorageMiner) {
-	go func() {
-		closing, err := nodeApi.Closing(ctx)
-		if err != nil {
-			log.Errorf("failed to get remote closing channel: %+v", err)
-		}
-
-		select {
-		case <-closing:
-		case <-ctx.Done():
-		}
-
-		if ctx.Err() != nil {
-			return // graceful shutdown
-		}
-
-		log.Warnf("Connection with miner node lost, restarting")
-
-		exe, err := os.Executable()
-		if err != nil {
-			log.Errorf("getting executable for auto-restart: %+v", err)
-		}
-
-		_ = log.Sync()
-
-		// TODO: there are probably cleaner/more graceful ways to restart,
-		//  but this is good enough for now (FSM can recover from the mess this creates)
-		//nolint:gosec
-		if err := syscall.Exec(exe, []string{exe,
-			fmt.Sprintf("--worker-repo=%s", cctx.String("worker-repo")),
-			fmt.Sprintf("--miner-repo=%s", cctx.String("miner-repo")),
-			fmt.Sprintf("--enable-gpu-proving=%t", cctx.Bool("enable-gpu-proving")),
-			"run",
-			fmt.Sprintf("--listen=%s", cctx.String("listen")),
-			fmt.Sprintf("--no-local-storage=%t", cctx.Bool("no-local-storage")),
-			fmt.Sprintf("--no-swap=%t", cctx.Bool("no-swap")),
-			fmt.Sprintf("--addpiece=%t", cctx.Bool("addpiece")),
-			fmt.Sprintf("--precommit1=%t", cctx.Bool("precommit1")),
-			fmt.Sprintf("--unseal=%t", cctx.Bool("unseal")),
-			fmt.Sprintf("--precommit2=%t", cctx.Bool("precommit2")),
-			fmt.Sprintf("--commit=%t", cctx.Bool("commit")),
-			fmt.Sprintf("--parallel-fetch-limit=%d", cctx.Int("parallel-fetch-limit")),
-			fmt.Sprintf("--timeout=%s", cctx.String("timeout")),
-		}, os.Environ()); err != nil {
-			fmt.Println(err)
-		}
-	}()
-}
-
 func extractRoutableIP(timeout time.Duration) (string, error) {
 	minerMultiAddrKey := "MINER_API_INFO"
 	deprecatedMinerMultiAddrKey := "STORAGE_API_INFO"

cmd/lotus-seal-worker/rpc.go

@@ -2,15 +2,16 @@ package main
 
 import (
 	"context"
+	"sync/atomic"
 
+	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/specs-storage/storage"
-
 	"github.com/filecoin-project/lotus/build"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )
 
 type worker struct {
@@ -18,6 +19,8 @@ type worker struct {
 
 	localStore *stores.Local
 	ls         stores.LocalStorage
+
+	disabled int64
 }
 
 func (w *worker) Version(context.Context) (build.Version, error) {
@@ -43,4 +46,34 @@ func (w *worker) StorageAddLocal(ctx context.Context, path string) error {
 	return nil
 }
 
-var _ storage.Sealer = &worker{}
+func (w *worker) SetEnabled(ctx context.Context, enabled bool) error {
+	disabled := int64(1)
+	if enabled {
+		disabled = 0
+	}
+	atomic.StoreInt64(&w.disabled, disabled)
+	return nil
+}
+
+func (w *worker) Enabled(ctx context.Context) (bool, error) {
+	return atomic.LoadInt64(&w.disabled) == 0, nil
+}
+
+func (w *worker) WaitQuiet(ctx context.Context) error {
+	w.LocalWorker.WaitQuiet() // uses WaitGroup under the hood so no ctx :/
+	return nil
+}
+
+func (w *worker) ProcessSession(ctx context.Context) (uuid.UUID, error) {
+	return w.LocalWorker.Session(ctx)
+}
+
+func (w *worker) Session(ctx context.Context) (uuid.UUID, error) {
+	if atomic.LoadInt64(&w.disabled) == 1 {
+		return uuid.UUID{}, xerrors.Errorf("worker disabled")
+	}
+
+	return w.LocalWorker.Session(ctx)
+}
+
+var _ storiface.WorkerCalls = &worker{}

cmd/lotus-seed/seed/seed.go

@@ -30,6 +30,7 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
+	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 	"github.com/filecoin-project/lotus/genesis"
 )
 
@@ -187,7 +188,7 @@ func presealSector(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, sid abi.Sector
 }
 
 func presealSectorFake(sbfs *basicfs.Provider, sid abi.SectorID, spt abi.RegisteredSealProof, ssize abi.SectorSize) (*genesis.PreSeal, error) {
-	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, 0, stores.FTSealed|stores.FTCache, stores.PathSealing)
+	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, 0, storiface.FTSealed|storiface.FTCache, storiface.PathSealing)
 	if err != nil {
 		return nil, xerrors.Errorf("acquire unsealed sector: %w", err)
 	}
@@ -211,7 +212,7 @@ func presealSectorFake(sbfs *basicfs.Provider, sid abi.SectorID, spt abi.Registe
 }
 
 func cleanupUnsealed(sbfs *basicfs.Provider, sid abi.SectorID) error {
-	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, stores.FTUnsealed, stores.FTNone, stores.PathSealing)
+	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
 	if err != nil {
 		return err
 	}

cmd/lotus-shed/balances.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"strconv"
 
 	"github.com/filecoin-project/lotus/chain/gen/genesis"
@@ -10,6 +11,7 @@ import (
 	_init "github.com/filecoin-project/lotus/chain/actors/builtin/init"
 
 	"github.com/docker/go-units"
+
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
@@ -24,6 +26,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/state"
@@ -33,7 +36,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/vm"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -168,19 +170,25 @@ var chainBalanceStateCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(repo.BlockstoreChain)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
 		mds, err := lkrepo.Datastore("/metadata")
 		if err != nil {
 			return err
 		}
 
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
 
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)
@@ -382,19 +390,25 @@ var chainPledgeCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(repo.BlockstoreChain)
 		if err != nil {
-			return err
+			return xerrors.Errorf("failed to open blockstore: %w", err)
 		}
 
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
 		mds, err := lkrepo.Datastore("/metadata")
 		if err != nil {
 			return err
 		}
 
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
 
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)

cmd/lotus-shed/cid.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+var cidCmd = &cli.Command{
+	Name: "cid",
+	Subcommands: cli.Commands{
+		cidIdCmd,
+	},
+}
+
+var cidIdCmd = &cli.Command{
+	Name:  "id",
+	Usage: "create identity CID from hex or base64 data",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify data")
+		}
+
+		dec, err := hex.DecodeString(cctx.Args().First())
+		if err != nil {
+			dec, err = base64.StdEncoding.DecodeString(cctx.Args().First())
+			if err != nil {
+				return err
+			}
+
+		}
+
+		builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.IDENTITY}
+
+		c, err := builder.Sum(dec)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(c)
+		return nil
+	},
+}

cmd/lotus-shed/datastore.go

@@ -1,17 +1,22 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
+	"github.com/dgraph-io/badger/v2"
 	"github.com/docker/go-units"
 	"github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
 	logging "github.com/ipfs/go-log"
+	"github.com/mitchellh/go-homedir"
 	"github.com/polydawn/refmt/cbor"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/lib/backupds"
@@ -25,6 +30,7 @@ var datastoreCmd = &cli.Command{
 		datastoreBackupCmd,
 		datastoreListCmd,
 		datastoreGetCmd,
+		datastoreRewriteCmd,
 	},
 }
 
@@ -288,3 +294,76 @@ func printVal(enc string, val []byte) error {
 
 	return nil
 }
+
+var datastoreRewriteCmd = &cli.Command{
+	Name:        "rewrite",
+	Description: "rewrites badger datastore to compact it and possibly change params",
+	ArgsUsage:   "source destination",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 2 {
+			return xerrors.Errorf("expected 2 arguments, got %d", cctx.NArg())
+		}
+		fromPath, err := homedir.Expand(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("cannot get fromPath: %w", err)
+		}
+		toPath, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("cannot get toPath: %w", err)
+		}
+
+		var (
+			from *badger.DB
+			to   *badger.DB
+		)
+
+		// open the destination (to) store.
+		opts, err := repo.BadgerBlockstoreOptions(repo.BlockstoreChain, toPath, false)
+		if err != nil {
+			return xerrors.Errorf("failed to get badger options: %w", err)
+		}
+		opts.SyncWrites = false
+		if to, err = badger.Open(opts.Options); err != nil {
+			return xerrors.Errorf("opening 'to' badger store: %w", err)
+		}
+
+		// open the source (from) store.
+		opts, err = repo.BadgerBlockstoreOptions(repo.BlockstoreChain, fromPath, true)
+		if err != nil {
+			return xerrors.Errorf("failed to get badger options: %w", err)
+		}
+		if from, err = badger.Open(opts.Options); err != nil {
+			return xerrors.Errorf("opening 'from' datastore: %w", err)
+		}
+
+		pr, pw := io.Pipe()
+		errCh := make(chan error)
+		go func() {
+			bw := bufio.NewWriterSize(pw, 64<<20)
+			_, err := from.Backup(bw, 0)
+			_ = bw.Flush()
+			_ = pw.CloseWithError(err)
+			errCh <- err
+		}()
+		go func() {
+			err := to.Load(pr, 256)
+			errCh <- err
+		}()
+
+		err = <-errCh
+		if err != nil {
+			select {
+			case nerr := <-errCh:
+				err = multierr.Append(err, nerr)
+			default:
+			}
+			return err
+		}
+
+		err = <-errCh
+		if err != nil {
+			return err
+		}
+		return multierr.Append(from.Close(), to.Close())
+	},
+}

cmd/lotus-shed/election.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var electionCmd = &cli.Command{
+	Name:  "election",
+	Usage: "commands related to leader election",
+	Subcommands: []*cli.Command{
+		electionRunDummy,
+	},
+}
+
+var electionRunDummy = &cli.Command{
+	Name:  "run-dummy",
+	Usage: "runs dummy elections with given power",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name: "network-power",
+		},
+		&cli.StringFlag{
+			Name: "miner-power",
+		},
+		&cli.Uint64Flag{
+			Name:  "seed",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		minerPow, err := types.BigFromString(cctx.String("miner-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding miner-power: %w", err)
+		}
+		networkPow, err := types.BigFromString(cctx.String("network-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding miner-power: %w", err)
+		}
+
+		ep := &types.ElectionProof{}
+		ep.VRFProof = make([]byte, 32)
+		seed := cctx.Uint64("seed")
+		if seed == 0 {
+			seed = rand.Uint64()
+		}
+		binary.BigEndian.PutUint64(ep.VRFProof, seed)
+
+		i := uint64(0)
+		for {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			binary.BigEndian.PutUint64(ep.VRFProof[8:], i)
+			j := ep.ComputeWinCount(minerPow, networkPow)
+			_, err := fmt.Printf("%t, %d\n", j != 0, j)
+			if err != nil {
+				return err
+			}
+			i++
+		}
+	},
+}

cmd/lotus-shed/export.go

@@ -3,16 +3,17 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -71,19 +72,25 @@ var exportChainCmd = &cli.Command{
 
 		defer fi.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(repo.BlockstoreChain)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
 		mds, err := lr.Datastore("/metadata")
 		if err != nil {
 			return err
 		}
 
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, nil, nil)
+		cs := store.NewChainStore(bs, bs, mds, nil, nil)
 		if err := cs.Load(); err != nil {
 			return err
 		}

cmd/lotus-shed/genesis-verify.go

@@ -52,7 +52,7 @@ var genesisVerifyCmd = &cli.Command{
 		}
 		bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
 
-		cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil)
+		cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil)
 
 		cf := cctx.Args().Get(0)
 		f, err := os.Open(cf)

cmd/lotus-shed/import-car.go

@@ -12,7 +12,6 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -45,12 +44,18 @@ var importCarCmd = &cli.Command{
 			return xerrors.Errorf("opening the car file: %w", err)
 		}
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(repo.BlockstoreChain)
 		if err != nil {
 			return err
 		}
 
-		bs := blockstore.NewBlockstore(ds)
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
 
 		cr, err := car.NewCarReader(f)
 		if err != nil {
@@ -65,7 +70,7 @@ var importCarCmd = &cli.Command{
 					return err
 				}
 				fmt.Println()
-				return ds.Close()
+				return nil
 			default:
 				if err := f.Close(); err != nil {
 					return err
@@ -108,12 +113,18 @@ var importObjectCmd = &cli.Command{
 		}
 		defer lr.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(repo.BlockstoreChain)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
-		bs := blockstore.NewBlockstore(ds)
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
 
 		c, err := cid.Decode(cctx.Args().Get(0))
 		if err != nil {

cmd/lotus-shed/main.go

@@ -46,6 +46,9 @@ func main() {
 		ledgerCmd,
 		sectorsCmd,
 		msgCmd,
+		electionCmd,
+		rpcCmd,
+		cidCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/pruning.go

@@ -3,20 +3,19 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/bbloom"
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/lotus/lib/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
 	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/ipfs/bbloom"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/query"
-	dshelp "github.com/ipfs/go-ipfs-ds-help"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
 )
 
 type cidSet interface {
@@ -132,12 +131,25 @@ var stateTreePruneCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(repo.BlockstoreChain)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
-		defer ds.Close() //nolint:errcheck
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		// After migrating to native blockstores, this has been made
+		// database-specific.
+		badgbs, ok := bs.(*badgerbs.Blockstore)
+		if !ok {
+			return fmt.Errorf("only badger blockstores are supported")
+		}
 
 		mds, err := lkrepo.Datastore("/metadata")
 		if err != nil {
@@ -145,24 +157,19 @@ var stateTreePruneCmd = &cli.Command{
 		}
 		defer mds.Close() //nolint:errcheck
 
+		const DiscardRatio = 0.2
 		if cctx.Bool("only-ds-gc") {
-			gcds, ok := ds.(datastore.GCDatastore)
-			if ok {
-				fmt.Println("running datastore gc....")
-				for i := 0; i < cctx.Int("gc-count"); i++ {
-					if err := gcds.CollectGarbage(); err != nil {
-						return xerrors.Errorf("datastore GC failed: %w", err)
-					}
+			fmt.Println("running datastore gc....")
+			for i := 0; i < cctx.Int("gc-count"); i++ {
+				if err := badgbs.DB.RunValueLogGC(DiscardRatio); err != nil {
+					return xerrors.Errorf("datastore GC failed: %w", err)
 				}
-				fmt.Println("gc complete!")
-				return nil
 			}
-			return fmt.Errorf("datastore doesnt support gc")
+			fmt.Println("gc complete!")
+			return nil
 		}
 
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
 		if err := cs.Load(); err != nil {
 			return fmt.Errorf("loading chainstore: %w", err)
 		}
@@ -199,63 +206,30 @@ var stateTreePruneCmd = &cli.Command{
 			return nil
 		}
 
-		var b datastore.Batch
-		var batchCount int
+		b := badgbs.DB.NewWriteBatch()
+		defer b.Cancel()
+
 		markForRemoval := func(c cid.Cid) error {
-			if b == nil {
-				nb, err := ds.Batch()
-				if err != nil {
-					return fmt.Errorf("opening batch: %w", err)
-				}
-
-				b = nb
-			}
-			batchCount++
-
-			if err := b.Delete(dshelp.MultihashToDsKey(c.Hash())); err != nil {
-				return err
-			}
-
-			if batchCount > 100 {
-				if err := b.Commit(); err != nil {
-					return xerrors.Errorf("failed to commit batch deletes: %w", err)
-				}
-				b = nil
-				batchCount = 0
-			}
-			return nil
+			return b.Delete(badgbs.StorageKey(nil, c))
 		}
 
-		res, err := ds.Query(query.Query{KeysOnly: true})
+		keys, err := bs.AllKeysChan(context.Background())
 		if err != nil {
-			return xerrors.Errorf("failed to query datastore: %w", err)
+			return xerrors.Errorf("failed to query blockstore: %w", err)
 		}
 
 		dupTo := cctx.Int("delete-up-to")
 
 		var deleteCount int
 		var goodHits int
-		for {
-			v, ok := res.NextSync()
-			if !ok {
-				break
-			}
-
-			bk, err := dshelp.BinaryFromDsKey(datastore.RawKey(v.Key[len("/blocks"):]))
-			if err != nil {
-				return xerrors.Errorf("failed to parse key: %w", err)
-			}
-
-			if goodSet.HasRaw(bk) {
+		for k := range keys {
+			if goodSet.HasRaw(k.Bytes()) {
 				goodHits++
 				continue
 			}
 
-			nc := cid.NewCidV1(cid.Raw, bk)
-
-			deleteCount++
-			if err := markForRemoval(nc); err != nil {
-				return fmt.Errorf("failed to remove cid %s: %w", nc, err)
+			if err := markForRemoval(k); err != nil {
+				return fmt.Errorf("failed to remove cid %s: %w", k, err)
 			}
 
 			if deleteCount%20 == 0 {
@@ -267,22 +241,17 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}
 
-		if b != nil {
-			if err := b.Commit(); err != nil {
-				return xerrors.Errorf("failed to commit final batch delete: %w", err)
-			}
+		if err := b.Flush(); err != nil {
+			return xerrors.Errorf("failed to flush final batch delete: %w", err)
 		}
 
-		gcds, ok := ds.(datastore.GCDatastore)
-		if ok {
-			fmt.Println("running datastore gc....")
-			for i := 0; i < cctx.Int("gc-count"); i++ {
-				if err := gcds.CollectGarbage(); err != nil {
-					return xerrors.Errorf("datastore GC failed: %w", err)
-				}
+		fmt.Println("running datastore gc....")
+		for i := 0; i < cctx.Int("gc-count"); i++ {
+			if err := badgbs.DB.RunValueLogGC(DiscardRatio); err != nil {
+				return xerrors.Errorf("datastore GC failed: %w", err)
 			}
-			fmt.Println("gc complete!")
 		}
+		fmt.Println("gc complete!")
 
 		return nil
 	},

cmd/lotus-shed/rpc.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"text/scanner"
+
+	"github.com/chzyer/readline"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var rpcCmd = &cli.Command{
+	Name:  "rpc",
+	Usage: "Interactive JsonPRC shell",
+	// TODO: flag for miner/worker
+	Action: func(cctx *cli.Context) error {
+		addr, headers, err := lcli.GetRawAPI(cctx, repo.FullNode)
+		if err != nil {
+			return err
+		}
+
+		u, err := url.Parse(addr)
+		if err != nil {
+			return xerrors.Errorf("parsing api URL: %w", err)
+		}
+
+		switch u.Scheme {
+		case "ws":
+			u.Scheme = "http"
+		case "wss":
+			u.Scheme = "https"
+		}
+
+		addr = u.String()
+
+		ctx := lcli.ReqContext(cctx)
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		afmt := lcli.NewAppFmt(cctx.App)
+
+		cs := readline.NewCancelableStdin(afmt.Stdin)
+		go func() {
+			<-ctx.Done()
+			cs.Close() // nolint:errcheck
+		}()
+
+		cctx.App.Metadata["repoType"] = repo.FullNode
+		if err := lcli.VersionCmd.Action(cctx); err != nil {
+			return err
+		}
+		fmt.Println("Usage: > Method [Param1, Param2, ...]")
+
+		rl, err := readline.NewEx(&readline.Config{
+			Stdin:             cs,
+			HistoryFile:       "/tmp/lotusrpc.tmp",
+			Prompt:            "> ",
+			EOFPrompt:         "exit",
+			HistorySearchFold: true,
+
+			// TODO: Some basic auto completion
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			line, err := rl.Readline()
+			if err == readline.ErrInterrupt {
+				if len(line) == 0 {
+					break
+				} else {
+					continue
+				}
+			} else if err == io.EOF {
+				break
+			}
+
+			var s scanner.Scanner
+			s.Init(strings.NewReader(line))
+			s.Scan()
+			method := s.TokenText()
+
+			s.Scan()
+			params := line[s.Position.Offset:]
+
+			jreq, err := json.Marshal(struct {
+				Jsonrpc string          `json:"jsonrpc"`
+				ID      int             `json:"id"`
+				Method  string          `json:"method"`
+				Params  json.RawMessage `json:"params"`
+			}{
+				Jsonrpc: "2.0",
+				Method:  "Filecoin." + method,
+				Params:  json.RawMessage(params),
+				ID:      0,
+			})
+			if err != nil {
+				return err
+			}
+
+			req, err := http.NewRequest("POST", addr, bytes.NewReader(jreq))
+			if err != nil {
+				return err
+			}
+			req.Header = headers
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+
+			rb, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(string(rb))
+
+			if err := resp.Body.Close(); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/sectors.go

@@ -25,6 +25,7 @@ var sectorsCmd = &cli.Command{
 	Flags: []cli.Flag{},
 	Subcommands: []*cli.Command{
 		terminateSectorCmd,
+		terminateSectorPenaltyEstimationCmd,
 	},
 }
 
@@ -131,3 +132,101 @@ var terminateSectorCmd = &cli.Command{
 		return nil
 	},
 }
+
+func findPenaltyInInternalExecutions(prefix string, trace []types.ExecutionTrace) {
+	for _, im := range trace {
+		if im.Msg.To.String() == "f099" /*Burn actor*/ {
+			fmt.Printf("Estimated termination penalty: %s attoFIL\n", im.Msg.Value)
+			return
+		}
+		findPenaltyInInternalExecutions(prefix+"\t", im.Subcalls)
+	}
+}
+
+var terminateSectorPenaltyEstimationCmd = &cli.Command{
+	Name:      "termination-estimate",
+	Usage:     "Estimate the termination penalty",
+	ArgsUsage: "[sectorNum1 sectorNum2 ...]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 1 {
+			return fmt.Errorf("at least one sector must be specified")
+		}
+
+		nodeApi, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		api, acloser, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		maddr, err := api.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		terminationDeclarationParams := []miner2.TerminationDeclaration{}
+
+		for _, sn := range cctx.Args().Slice() {
+			sectorNum, err := strconv.ParseUint(sn, 10, 64)
+			if err != nil {
+				return fmt.Errorf("could not parse sector number: %w", err)
+			}
+
+			sectorbit := bitfield.New()
+			sectorbit.Set(sectorNum)
+
+			loca, err := nodeApi.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK)
+			if err != nil {
+				return fmt.Errorf("get state sector partition %s", err)
+			}
+
+			para := miner2.TerminationDeclaration{
+				Deadline:  loca.Deadline,
+				Partition: loca.Partition,
+				Sectors:   sectorbit,
+			}
+
+			terminationDeclarationParams = append(terminationDeclarationParams, para)
+		}
+
+		terminateSectorParams := &miner2.TerminateSectorsParams{
+			Terminations: terminationDeclarationParams,
+		}
+
+		sp, err := actors.SerializeParams(terminateSectorParams)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		msg := &types.Message{
+			From:   mi.Owner,
+			To:     maddr,
+			Method: miner.Methods.TerminateSectors,
+
+			Value:  big.Zero(),
+			Params: sp,
+		}
+
+		//TODO: 4667 add an option to give a more precise estimation with pending termination penalty excluded
+
+		invocResult, err := nodeApi.StateCall(ctx, msg, types.TipSetKey{})
+		if err != nil {
+			return xerrors.Errorf("fail to state call: %w", err)
+		}
+
+		findPenaltyInInternalExecutions("\t", invocResult.ExecutionTrace.Subcalls)
+		return nil
+	},
+}

cmd/lotus-stats/docker-compose.yml

@@ -4,10 +4,10 @@ services:
   influxdb:
     image: influxdb:latest
     container_name: influxdb
+    ports:
+      - "18086:8086"
     environment:
       - INFLUXDB_DB=lotus
-    ports:
-      - "8086:8086"
     volumes:
       - influxdb:/var/lib/influxdb
 
@@ -15,7 +15,7 @@ services:
     image: grafana/grafana:latest
     container_name: grafana
     ports:
-      - "3000:3000"
+      - "13000:3000"
     links:
       - influxdb
     volumes:

cmd/lotus-stats/env.stats

@@ -1,3 +1,3 @@
-export INFLUX_ADDR="http://localhost:8086"
+export INFLUX_ADDR="http://localhost:18086"
 export INFLUX_USER=""
 export INFLUX_PASS=""

cmd/lotus-stats/main.go

@@ -2,71 +2,158 @@ package main
 
 import (
 	"context"
-	"flag"
 	"os"
 
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/tools/stats"
+
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
 )
 
 var log = logging.Logger("stats")
 
-const (
-	influxAddrEnvVar = "INFLUX_ADDR"
-	influxUserEnvVar = "INFLUX_USER"
-	influxPassEnvVar = "INFLUX_PASS"
-)
-
 func main() {
-	var repo string = "~/.lotus"
-	var database string = "lotus"
-	var reset bool = false
-	var nosync bool = false
-	var height int64 = 0
-	var headlag int = 3
-
-	flag.StringVar(&repo, "repo", repo, "lotus repo path")
-	flag.StringVar(&database, "database", database, "influx database")
-	flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)")
-	flag.IntVar(&headlag, "head-lag", headlag, "number of head events to hold to protect against small reorgs")
-	flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering")
-	flag.BoolVar(&nosync, "nosync", nosync, "skip waiting for sync")
-
-	flag.Parse()
-
-	ctx := context.Background()
-
-	influx, err := stats.InfluxClient(os.Getenv(influxAddrEnvVar), os.Getenv(influxUserEnvVar), os.Getenv(influxPassEnvVar))
-	if err != nil {
-		log.Fatal(err)
+	local := []*cli.Command{
+		runCmd,
+		versionCmd,
 	}
 
-	if reset {
-		if err := stats.ResetDatabase(influx, database); err != nil {
-			log.Fatal(err)
-		}
+	app := &cli.App{
+		Name:    "lotus-stats",
+		Usage:   "Collect basic information about a filecoin network using lotus",
+		Version: build.UserVersion(),
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "lotus-path",
+				EnvVars: []string{"LOTUS_PATH"},
+				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
+			},
+			&cli.StringFlag{
+				Name:    "log-level",
+				EnvVars: []string{"LOTUS_STATS_LOG_LEVEL"},
+				Value:   "info",
+			},
+		},
+		Before: func(cctx *cli.Context) error {
+			return logging.SetLogLevel("stats", cctx.String("log-level"))
+		},
+		Commands: local,
 	}
 
-	if !reset && height == 0 {
-		h, err := stats.GetLastRecordedHeight(influx, database)
-		if err != nil {
-			log.Info(err)
-		}
-
-		height = h
+	if err := app.Run(os.Args); err != nil {
+		log.Errorw("exit in error", "err", err)
+		os.Exit(1)
+		return
 	}
+}
 
-	api, closer, err := stats.GetFullNodeAPI(ctx, repo)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer closer()
+var versionCmd = &cli.Command{
+	Name:  "version",
+	Usage: "Print version",
+	Action: func(cctx *cli.Context) error {
+		cli.VersionPrinter(cctx)
+		return nil
+	},
+}
 
-	if !nosync {
-		if err := stats.WaitForSyncComplete(ctx, api); err != nil {
+var runCmd = &cli.Command{
+	Name:  "run",
+	Usage: "",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "influx-database",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_DATABASE"},
+			Usage:   "influx database",
+			Value:   "",
+		},
+		&cli.StringFlag{
+			Name:    "influx-hostname",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_HOSTNAME"},
+			Value:   "http://localhost:8086",
+			Usage:   "influx hostname",
+		},
+		&cli.StringFlag{
+			Name:    "influx-username",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_USERNAME"},
+			Usage:   "influx username",
+			Value:   "",
+		},
+		&cli.StringFlag{
+			Name:    "influx-password",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_PASSWORD"},
+			Usage:   "influx password",
+			Value:   "",
+		},
+		&cli.IntFlag{
+			Name:    "height",
+			EnvVars: []string{"LOTUS_STATS_HEIGHT"},
+			Usage:   "tipset height to start processing from",
+			Value:   0,
+		},
+		&cli.IntFlag{
+			Name:    "head-lag",
+			EnvVars: []string{"LOTUS_STATS_HEAD_LAG"},
+			Usage:   "the number of tipsets to delay processing on to smooth chain reorgs",
+			Value:   int(build.MessageConfidence),
+		},
+		&cli.BoolFlag{
+			Name:    "no-sync",
+			EnvVars: []string{"LOTUS_STATS_NO_SYNC"},
+			Usage:   "do not wait for chain sync to complete",
+			Value:   false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.Background()
+
+		resetFlag := cctx.Bool("reset")
+		noSyncFlag := cctx.Bool("no-sync")
+		heightFlag := cctx.Int("height")
+		headLagFlag := cctx.Int("head-lag")
+
+		influxAddrFlag := cctx.String("influx-addr")
+		influxUserFlag := cctx.String("influx-user")
+		influxPassFlag := cctx.String("influx-pass")
+		influxDatabaseFlag := cctx.String("influx-database")
+
+		influx, err := stats.InfluxClient(influxAddrFlag, influxUserFlag, influxPassFlag)
+		if err != nil {
 			log.Fatal(err)
+		}
+
+		if resetFlag {
+			if err := stats.ResetDatabase(influx, influxDatabaseFlag); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		height := int64(heightFlag)
+
+		if !resetFlag && height == 0 {
+			h, err := stats.GetLastRecordedHeight(influx, influxDatabaseFlag)
+			if err != nil {
+				log.Info(err)
+			}
+
+			height = h
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
 		}
-	}
+		defer closer()
+
+		if !noSyncFlag {
+			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		stats.Collect(ctx, api, influx, influxDatabaseFlag, height, headLagFlag)
 
-	stats.Collect(ctx, api, influx, database, height, headlag)
+		return nil
+	},
 }

cmd/lotus-stats/setup.bash

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 
-GRAFANA_HOST="localhost:3000"
+GRAFANA_HOST="http://localhost:13000"
 
 curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF
 {
-  "name":"InfluxDB",
+  "name":"filecoin-ntwk-localstats",
   "type":"influxdb",
   "database":"lotus",
   "url": "http://influxdb:8086",