Merge branch 'master' into releases

.circleci/config.yml

@@ -408,9 +408,18 @@ jobs:
     steps:
       - install-deps
       - prepare
+      - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
+      - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
+      - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
       - run: make deps
       - run: make docsgen
+      - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
+      - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
+      - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
       - run: git --no-pager diff
+      - run: diff ../pre-openrpc-full ../post-openrpc-full
+      - run: diff ../pre-openrpc-miner ../post-openrpc-miner
+      - run: diff ../pre-openrpc-worker ../post-openrpc-worker
       - run: git --no-pager diff --quiet
 
   lint: &lint

.gitignore

@@ -14,6 +14,8 @@
 /lotus-pcr
 /lotus-wallet
 /lotus-keygen
+/docgen-md
+/docgen-openrpc
 /bench.json
 /lotuspond/front/node_modules
 /lotuspond/front/build

Makefile

@@ -325,12 +325,41 @@ type-gen:
 method-gen:
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
-gen: type-gen method-gen
+api-gen:
+	go run ./gen/api > api/apistruct/struct.go
+	goimports -w api/apistruct
+	goimports -w api/apistruct
+.PHONY: api-gen
 
-docsgen:
+docsgen: docsgen-md docsgen-openrpc
-	go run ./api/docgen "api/api_full.go" "FullNode" > documentation/en/api-methods.md
+
-	go run ./api/docgen "api/api_storage.go" "StorageMiner" > documentation/en/api-methods-miner.md
+docsgen-md-bin:
-	go run ./api/docgen "api/api_worker.go" "WorkerAPI" > documentation/en/api-methods-worker.md
+	go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
+docsgen-openrpc-bin:
+	go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
+
+docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
+
+docsgen-md-full: docsgen-md-bin
+	./docgen-md "api/api_full.go" "FullNode" > documentation/en/api-methods.md
+docsgen-md-storage: docsgen-md-bin
+	./docgen-md "api/api_storage.go" "StorageMiner" > documentation/en/api-methods-miner.md
+docsgen-md-worker: docsgen-md-bin
+	./docgen-md "api/api_worker.go" "Worker" > documentation/en/api-methods-worker.md
+
+docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
+
+docsgen-openrpc-full: docsgen-openrpc-bin
+	./docgen-openrpc "api/api_full.go" "FullNode" -gzip > build/openrpc/full.json.gz
+docsgen-openrpc-storage: docsgen-openrpc-bin
+	./docgen-openrpc "api/api_storage.go" "StorageMiner" -gzip > build/openrpc/miner.json.gz
+docsgen-openrpc-worker: docsgen-openrpc-bin
+	./docgen-openrpc "api/api_worker.go" "Worker" -gzip > build/openrpc/worker.json.gz
+
+.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
+
+gen: type-gen method-gen docsgen api-gen
+.PHONY: gen
 
 print-%:
 	@echo $*=$($*)

api/api_common.go

@@ -11,63 +11,68 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	protocol "github.com/libp2p/go-libp2p-core/protocol"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
 )
 
 type Common interface {
 
 	// MethodGroup: Auth
 
-	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error)
+	AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) //perm:read
-	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)
+	AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error)    //perm:admin
 
 	// MethodGroup: Net
 
-	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
+	NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) //perm:read
-	NetPeers(context.Context) ([]peer.AddrInfo, error)
+	NetPeers(context.Context) ([]peer.AddrInfo, error)                        //perm:read
-	NetConnect(context.Context, peer.AddrInfo) error
+	NetConnect(context.Context, peer.AddrInfo) error                          //perm:write
-	NetAddrsListen(context.Context) (peer.AddrInfo, error)
+	NetAddrsListen(context.Context) (peer.AddrInfo, error)                    //perm:read
-	NetDisconnect(context.Context, peer.ID) error
+	NetDisconnect(context.Context, peer.ID) error                             //perm:write
-	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
+	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)              //perm:read
-	NetPubsubScores(context.Context) ([]PubsubScore, error)
+	NetPubsubScores(context.Context) ([]PubsubScore, error)                   //perm:read
-	NetAutoNatStatus(context.Context) (NatInfo, error)
+	NetAutoNatStatus(context.Context) (NatInfo, error)                        //perm:read
-	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
+	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)           //perm:read
-	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)
+	NetPeerInfo(context.Context, peer.ID) (*ExtendedPeerInfo, error)          //perm:read
 
 	// NetBandwidthStats returns statistics about the nodes total bandwidth
 	// usage and current rate across all peers and protocols.
-	NetBandwidthStats(ctx context.Context) (metrics.Stats, error)
+	NetBandwidthStats(ctx context.Context) (metrics.Stats, error) //perm:read
 
 	// NetBandwidthStatsByPeer returns statistics about the nodes bandwidth
 	// usage and current rate per peer
-	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error)
+	NetBandwidthStatsByPeer(ctx context.Context) (map[string]metrics.Stats, error) //perm:read
 
 	// NetBandwidthStatsByProtocol returns statistics about the nodes bandwidth
 	// usage and current rate per protocol
-	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error)
+	NetBandwidthStatsByProtocol(ctx context.Context) (map[protocol.ID]metrics.Stats, error) //perm:read
 
 	// ConnectionGater API
-	NetBlockAdd(ctx context.Context, acl NetBlockList) error
+	NetBlockAdd(ctx context.Context, acl NetBlockList) error    //perm:admin
-	NetBlockRemove(ctx context.Context, acl NetBlockList) error
+	NetBlockRemove(ctx context.Context, acl NetBlockList) error //perm:admin
-	NetBlockList(ctx context.Context) (NetBlockList, error)
+	NetBlockList(ctx context.Context) (NetBlockList, error)     //perm:read
 
 	// MethodGroup: Common
 
+	// Discover returns an OpenRPC document describing an RPC API.
+	Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
+
 	// ID returns peerID of libp2p node backing this API
-	ID(context.Context) (peer.ID, error)
+	ID(context.Context) (peer.ID, error) //perm:read
 
 	// Version provides information about API provider
-	Version(context.Context) (APIVersion, error)
+	Version(context.Context) (APIVersion, error) //perm:read
 
-	LogList(context.Context) ([]string, error)
+	LogList(context.Context) ([]string, error)         //perm:write
-	LogSetLevel(context.Context, string, string) error
+	LogSetLevel(context.Context, string, string) error //perm:write
 
 	// trigger graceful shutdown
-	Shutdown(context.Context) error
+	Shutdown(context.Context) error //perm:admin
 
 	// Session returns a random UUID of api provider session
-	Session(context.Context) (uuid.UUID, error)
+	Session(context.Context) (uuid.UUID, error) //perm:read
 
-	Closing(context.Context) (<-chan struct{}, error)
+	Closing(context.Context) (<-chan struct{}, error) //perm:read
 }
 
 // APIVersion provides various build-time information

api/api_full.go

@@ -6,14 +6,12 @@ import (
 	"fmt"
 	"time"
 
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-state-types/network"
-
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-multistore"
@@ -21,11 +19,12 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
-
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
@@ -50,66 +49,78 @@ type FullNode interface {
 
 	// ChainNotify returns channel with chain head updates.
 	// First message is guaranteed to be of len == 1, and type == 'current'.
-	ChainNotify(context.Context) (<-chan []*HeadChange, error)
+	ChainNotify(context.Context) (<-chan []*HeadChange, error) //perm:read
 
 	// ChainHead returns the current head of the chain.
-	ChainHead(context.Context) (*types.TipSet, error)
+	ChainHead(context.Context) (*types.TipSet, error) //perm:read
 
 	// ChainGetRandomnessFromTickets is used to sample the chain for randomness.
-	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
+	ChainGetRandomnessFromTickets(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
 
 	// ChainGetRandomnessFromBeacon is used to sample the beacon for randomness.
-	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
+	ChainGetRandomnessFromBeacon(ctx context.Context, tsk types.TipSetKey, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) //perm:read
 
 	// ChainGetBlock returns the block specified by the given CID.
-	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
+	ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error) //perm:read
 	// ChainGetTipSet returns the tipset specified by the given TipSetKey.
-	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
+	ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error) //perm:read
 
 	// ChainGetBlockMessages returns messages stored in the specified block.
-	ChainGetBlockMessages(ctx context.Context, blockCid cid.Cid) (*BlockMessages, error)
+	//
+	// Note: If there are multiple blocks in a tipset, it's likely that some
+	// messages will be duplicated. It's also possible for blocks in a tipset to have
+	// different messages from the same sender at the same nonce. When that happens,
+	// only the first message (in a block with lowest ticket) will be considered
+	// for execution
+	//
+	// NOTE: THIS METHOD SHOULD ONLY BE USED FOR GETTING MESSAGES IN A SPECIFIC BLOCK
+	//
+	// DO NOT USE THIS METHOD TO GET MESSAGES INCLUDED IN A TIPSET
+	// Use ChainGetParentMessages, which will perform correct message deduplication
+	ChainGetBlockMessages(ctx context.Context, blockCid cid.Cid) (*BlockMessages, error) //perm:read
 
 	// ChainGetParentReceipts returns receipts for messages in parent tipset of
-	// the specified block.
+	// the specified block. The receipts in the list returned is one-to-one with the
-	ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error)
+	// messages returned by a call to ChainGetParentMessages with the same blockCid.
+	ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error) //perm:read
 
 	// ChainGetParentMessages returns messages stored in parent tipset of the
 	// specified block.
-	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]Message, error)
+	ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]Message, error) //perm:read
 
 	// ChainGetTipSetByHeight looks back for a tipset at the specified epoch.
 	// If there are no blocks at the specified epoch, a tipset at an earlier epoch
 	// will be returned.
-	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error)
+	ChainGetTipSetByHeight(context.Context, abi.ChainEpoch, types.TipSetKey) (*types.TipSet, error) //perm:read
 
 	// ChainReadObj reads ipld nodes referenced by the specified CID from chain
 	// blockstore and returns raw bytes.
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error) //perm:read
 
 	// ChainDeleteObj deletes node referenced by the given CID
-	ChainDeleteObj(context.Context, cid.Cid) error
+	ChainDeleteObj(context.Context, cid.Cid) error //perm:admin
 
 	// ChainHasObj checks if a given CID exists in the chain blockstore.
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
+	ChainHasObj(context.Context, cid.Cid) (bool, error) //perm:read
 
 	// ChainStatObj returns statistics about the graph referenced by 'obj'.
 	// If 'base' is also specified, then the returned stat will be a diff
 	// between the two objects.
-	ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (ObjStat, error)
+	ChainStatObj(ctx context.Context, obj cid.Cid, base cid.Cid) (ObjStat, error) //perm:read
 
 	// ChainSetHead forcefully sets current chain head. Use with caution.
-	ChainSetHead(context.Context, types.TipSetKey) error
+	ChainSetHead(context.Context, types.TipSetKey) error //perm:admin
 
 	// ChainGetGenesis returns the genesis tipset.
-	ChainGetGenesis(context.Context) (*types.TipSet, error)
+	ChainGetGenesis(context.Context) (*types.TipSet, error) //perm:read
 
 	// ChainTipSetWeight computes weight for the specified tipset.
-	ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error)
+	ChainTipSetWeight(context.Context, types.TipSetKey) (types.BigInt, error) //perm:read
-	ChainGetNode(ctx context.Context, p string) (*IpldObject, error)
+	ChainGetNode(ctx context.Context, p string) (*IpldObject, error)          //perm:read
 
 	// ChainGetMessage reads a message referenced by the specified CID from the
 	// chain blockstore.
-	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error)
+	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error) //perm:read
 
 	// ChainGetPath returns a set of revert/apply operations needed to get from
 	// one tipset to another, for example:
@@ -124,14 +135,14 @@ type FullNode interface {
 	//     tRR
 	//```
 	// Would return `[revert(tBA), apply(tAB), apply(tAA)]`
-	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error)
+	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) //perm:read
 
 	// ChainExport returns a stream of bytes with CAR dump of chain data.
 	// The exported chain data includes the header chain from the given tipset
 	// back to genesis, the entire genesis state, and the most recent 'nroots'
 	// state trees.
 	// If oldmsgskip is set, messages from before the requested roots are also not included.
-	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error)
+	ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read
 
 	// MethodGroup: Beacon
 	// The Beacon method group contains methods for interacting with the random beacon (DRAND)
@@ -139,74 +150,74 @@ type FullNode interface {
 	// BeaconGetEntry returns the beacon entry for the given filecoin epoch. If
 	// the entry has not yet been produced, the call will block until the entry
 	// becomes available
-	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error)
+	BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) //perm:read
 
 	// GasEstimateFeeCap estimates gas fee cap
-	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error)
+	GasEstimateFeeCap(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) //perm:read
 
 	// GasEstimateGasLimit estimates gas used by the message and returns it.
 	// It fails if message fails to execute.
-	GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error)
+	GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error) //perm:read
 
 	// GasEstimateGasPremium estimates what gas price should be used for a
 	// message to have high likelihood of inclusion in `nblocksincl` epochs.
 
 	GasEstimateGasPremium(_ context.Context, nblocksincl uint64,
-		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error)
+		sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) //perm:read
 
 	// GasEstimateMessageGas estimates gas values for unset message gas fields
-	GasEstimateMessageGas(context.Context, *types.Message, *MessageSendSpec, types.TipSetKey) (*types.Message, error)
+	GasEstimateMessageGas(context.Context, *types.Message, *MessageSendSpec, types.TipSetKey) (*types.Message, error) //perm:read
 
 	// MethodGroup: Sync
 	// The Sync method group contains methods for interacting with and
 	// observing the lotus sync service.
 
 	// SyncState returns the current status of the lotus sync system.
-	SyncState(context.Context) (*SyncState, error)
+	SyncState(context.Context) (*SyncState, error) //perm:read
 
 	// SyncSubmitBlock can be used to submit a newly created block to the.
 	// network through this node
-	SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error
+	SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error //perm:write
 
 	// SyncIncomingBlocks returns a channel streaming incoming, potentially not
 	// yet synced block headers.
-	SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error)
+	SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error) //perm:read
 
 	// SyncCheckpoint marks a blocks as checkpointed, meaning that it won't ever fork away from it.
-	SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error
+	SyncCheckpoint(ctx context.Context, tsk types.TipSetKey) error //perm:admin
 
 	// SyncMarkBad marks a blocks as bad, meaning that it won't ever by synced.
 	// Use with extreme caution.
-	SyncMarkBad(ctx context.Context, bcid cid.Cid) error
+	SyncMarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
 
 	// SyncUnmarkBad unmarks a blocks as bad, making it possible to be validated and synced again.
-	SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error
+	SyncUnmarkBad(ctx context.Context, bcid cid.Cid) error //perm:admin
 
 	// SyncUnmarkAllBad purges bad block cache, making it possible to sync to chains previously marked as bad
-	SyncUnmarkAllBad(ctx context.Context) error
+	SyncUnmarkAllBad(ctx context.Context) error //perm:admin
 
 	// SyncCheckBad checks if a block was marked as bad, and if it was, returns
 	// the reason.
-	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error)
+	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error) //perm:read
 
 	// SyncValidateTipset indicates whether the provided tipset is valid or not
-	SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error)
+	SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) //perm:read
 
 	// MethodGroup: Mpool
 	// The Mpool methods are for interacting with the message pool. The message pool
 	// manages all incoming and outgoing 'messages' going over the network.
 
 	// MpoolPending returns pending mempool messages.
-	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error)
+	MpoolPending(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) //perm:read
 
 	// MpoolSelect returns a list of pending messages for inclusion in the next block
-	MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error)
+	MpoolSelect(context.Context, types.TipSetKey, float64) ([]*types.SignedMessage, error) //perm:read
 
 	// MpoolPush pushes a signed message to mempool.
-	MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error)
+	MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
 
 	// MpoolPushUntrusted pushes a signed message to mempool from untrusted sources.
-	MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error)
+	MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error) //perm:write
 
 	// MpoolPushMessage atomically assigns a nonce, signs, and pushes a message
 	// to mempool.
@@ -214,34 +225,34 @@ type FullNode interface {
 	//
 	// When maxFee is set to 0, MpoolPushMessage will guess appropriate fee
 	// based on current chain conditions
-	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *MessageSendSpec) (*types.SignedMessage, error)
+	MpoolPushMessage(ctx context.Context, msg *types.Message, spec *MessageSendSpec) (*types.SignedMessage, error) //perm:sign
 
 	// MpoolBatchPush batch pushes a signed message to mempool.
-	MpoolBatchPush(context.Context, []*types.SignedMessage) ([]cid.Cid, error)
+	MpoolBatchPush(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
 
 	// MpoolBatchPushUntrusted batch pushes a signed message to mempool from untrusted sources.
-	MpoolBatchPushUntrusted(context.Context, []*types.SignedMessage) ([]cid.Cid, error)
+	MpoolBatchPushUntrusted(context.Context, []*types.SignedMessage) ([]cid.Cid, error) //perm:write
 
 	// MpoolBatchPushMessage batch pushes a unsigned message to mempool.
-	MpoolBatchPushMessage(context.Context, []*types.Message, *MessageSendSpec) ([]*types.SignedMessage, error)
+	MpoolBatchPushMessage(context.Context, []*types.Message, *MessageSendSpec) ([]*types.SignedMessage, error) //perm:sign
 
 	// MpoolGetNonce gets next nonce for the specified sender.
 	// Note that this method may not be atomic. Use MpoolPushMessage instead.
-	MpoolGetNonce(context.Context, address.Address) (uint64, error)
+	MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
-	MpoolSub(context.Context) (<-chan MpoolUpdate, error)
+	MpoolSub(context.Context) (<-chan MpoolUpdate, error)           //perm:read
 
 	// MpoolClear clears pending messages from the mpool
-	MpoolClear(context.Context, bool) error
+	MpoolClear(context.Context, bool) error //perm:write
 
 	// MpoolGetConfig returns (a copy of) the current mpool config
-	MpoolGetConfig(context.Context) (*types.MpoolConfig, error)
+	MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read
 	// MpoolSetConfig sets the mpool config to (a copy of) the supplied config
-	MpoolSetConfig(context.Context, *types.MpoolConfig) error
+	MpoolSetConfig(context.Context, *types.MpoolConfig) error //perm:admin
 
 	// MethodGroup: Miner
 
-	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*MiningBaseInfo, error)
+	MinerGetBaseInfo(context.Context, address.Address, abi.ChainEpoch, types.TipSetKey) (*MiningBaseInfo, error) //perm:read
-	MinerCreateBlock(context.Context, *BlockTemplate) (*types.BlockMsg, error)
+	MinerCreateBlock(context.Context, *BlockTemplate) (*types.BlockMsg, error)                                   //perm:write
 
 	// // UX ?
 
@@ -250,32 +261,32 @@ type FullNode interface {
 	// WalletNew creates a new address in the wallet with the given sigType.
 	// Available key types: bls, secp256k1, secp256k1-ledger
 	// Support for numerical types: 1 - secp256k1, 2 - BLS is deprecated
-	WalletNew(context.Context, types.KeyType) (address.Address, error)
+	WalletNew(context.Context, types.KeyType) (address.Address, error) //perm:write
 	// WalletHas indicates whether the given address is in the wallet.
-	WalletHas(context.Context, address.Address) (bool, error)
+	WalletHas(context.Context, address.Address) (bool, error) //perm:write
 	// WalletList lists all the addresses in the wallet.
-	WalletList(context.Context) ([]address.Address, error)
+	WalletList(context.Context) ([]address.Address, error) //perm:write
 	// WalletBalance returns the balance of the given address at the current head of the chain.
-	WalletBalance(context.Context, address.Address) (types.BigInt, error)
+	WalletBalance(context.Context, address.Address) (types.BigInt, error) //perm:read
 	// WalletSign signs the given bytes using the given address.
-	WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error)
+	WalletSign(context.Context, address.Address, []byte) (*crypto.Signature, error) //perm:sign
 	// WalletSignMessage signs the given message using the given address.
-	WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error)
+	WalletSignMessage(context.Context, address.Address, *types.Message) (*types.SignedMessage, error) //perm:sign
 	// WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid.
 	// The address does not have to be in the wallet.
-	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error)
+	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) //perm:read
 	// WalletDefaultAddress returns the address marked as default in the wallet.
-	WalletDefaultAddress(context.Context) (address.Address, error)
+	WalletDefaultAddress(context.Context) (address.Address, error) //perm:write
 	// WalletSetDefault marks the given address as as the default one.
-	WalletSetDefault(context.Context, address.Address) error
+	WalletSetDefault(context.Context, address.Address) error //perm:write
 	// WalletExport returns the private key of an address in the wallet.
-	WalletExport(context.Context, address.Address) (*types.KeyInfo, error)
+	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin
 	// WalletImport receives a KeyInfo, which includes a private key, and imports it into the wallet.
-	WalletImport(context.Context, *types.KeyInfo) (address.Address, error)
+	WalletImport(context.Context, *types.KeyInfo) (address.Address, error) //perm:admin
 	// WalletDelete deletes an address from the wallet.
-	WalletDelete(context.Context, address.Address) error
+	WalletDelete(context.Context, address.Address) error //perm:admin
 	// WalletValidateAddress validates whether a given string can be decoded as a well-formed address
-	WalletValidateAddress(context.Context, string) (address.Address, error)
+	WalletValidateAddress(context.Context, string) (address.Address, error) //perm:read
 
 	// Other
 
@@ -284,56 +295,56 @@ type FullNode interface {
 	// retrieval markets as a client
 
 	// ClientImport imports file under the specified path into filestore.
-	ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error)
+	ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin
 	// ClientRemoveImport removes file import
-	ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error
+	ClientRemoveImport(ctx context.Context, importID multistore.StoreID) error //perm:admin
 	// ClientStartDeal proposes a deal with a miner.
-	ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
+	ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin
 	// ClientGetDealInfo returns the latest information about a given deal.
-	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
+	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error) //perm:read
 	// ClientListDeals returns information about the deals made by the local client.
-	ClientListDeals(ctx context.Context) ([]DealInfo, error)
+	ClientListDeals(ctx context.Context) ([]DealInfo, error) //perm:write
 	// ClientGetDealUpdates returns the status of updated deals
-	ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error)
+	ClientGetDealUpdates(ctx context.Context) (<-chan DealInfo, error) //perm:write
 	// ClientGetDealStatus returns status given a code
-	ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error)
+	ClientGetDealStatus(ctx context.Context, statusCode uint64) (string, error) //perm:read
 	// ClientHasLocal indicates whether a certain CID is locally stored.
-	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
+	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) //perm:write
 	// ClientFindData identifies peers that have a certain file, and returns QueryOffers (one per peer).
-	ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]QueryOffer, error)
+	ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid) ([]QueryOffer, error) //perm:read
 	// ClientMinerQueryOffer returns a QueryOffer for the specific miner and file.
-	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error)
+	ClientMinerQueryOffer(ctx context.Context, miner address.Address, root cid.Cid, piece *cid.Cid) (QueryOffer, error) //perm:read
 	// ClientRetrieve initiates the retrieval of a file, as specified in the order.
-	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error
+	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref *FileRef) error //perm:admin
 	// ClientRetrieveWithEvents initiates the retrieval of a file, as specified in the order, and provides a channel
 	// of status updates.
-	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error)
+	ClientRetrieveWithEvents(ctx context.Context, order RetrievalOrder, ref *FileRef) (<-chan marketevents.RetrievalEvent, error) //perm:admin
 	// ClientQueryAsk returns a signed StorageAsk from the specified miner.
-	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error)
+	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.StorageAsk, error) //perm:read
 	// ClientCalcCommP calculates the CommP and data size of the specified CID
-	ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error)
+	ClientDealPieceCID(ctx context.Context, root cid.Cid) (DataCIDSize, error) //perm:read
 	// ClientCalcCommP calculates the CommP for a specified file
-	ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error)
+	ClientCalcCommP(ctx context.Context, inpath string) (*CommPRet, error) //perm:write
 	// ClientGenCar generates a CAR file for the specified file.
-	ClientGenCar(ctx context.Context, ref FileRef, outpath string) error
+	ClientGenCar(ctx context.Context, ref FileRef, outpath string) error //perm:write
 	// ClientDealSize calculates real deal data size
-	ClientDealSize(ctx context.Context, root cid.Cid) (DataSize, error)
+	ClientDealSize(ctx context.Context, root cid.Cid) (DataSize, error) //perm:read
 	// ClientListTransfers returns the status of all ongoing transfers of data
-	ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
+	ClientListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)        //perm:write
-	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
+	ClientDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error) //perm:write
 	// ClientRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	ClientRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// ClientCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	ClientCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// ClientRetrieveTryRestartInsufficientFunds attempts to restart stalled retrievals on a given payment channel
 	// which are stuck due to insufficient funds
-	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error
+	ClientRetrieveTryRestartInsufficientFunds(ctx context.Context, paymentChannel address.Address) error //perm:write
 
 	// ClientUnimport removes references to the specified file from filestore
 	//ClientUnimport(path string)
 
 	// ClientListImports lists imported files and their root CIDs
-	ClientListImports(ctx context.Context) ([]Import, error)
+	ClientListImports(ctx context.Context) ([]Import, error) //perm:write
 
 	//ClientListAsks() []Ask
 
@@ -347,149 +358,258 @@ type FullNode interface {
 	// StateCall applies the message to the tipset's parent state. The
 	// message is not applied on-top-of the messages in the passed-in
 	// tipset.
-	StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error)
+	StateCall(context.Context, *types.Message, types.TipSetKey) (*InvocResult, error) //perm:read
 	// StateReplay replays a given message, assuming it was included in a block in the specified tipset.
-	// If no tipset key is provided, the appropriate tipset is looked up.
+	//
-	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error)
+	// If a tipset key is provided, and a replacing message is found on chain,
+	// the method will return an error saying that the message wasn't found
+	//
+	// If no tipset key is provided, the appropriate tipset is looked up, and if
+	// the message was gas-repriced, the on-chain message will be replayed - in
+	// that case the returned InvocResult.MsgCid will not match the Cid param
+	//
+	// If the caller wants to ensure that exactly the requested message was executed,
+	// they MUST check that InvocResult.MsgCid is equal to the provided Cid.
+	// Without this check both the requested and original message may appear as
+	// successfully executed on-chain, which may look like a double-spend.
+	//
+	// A replacing message is a message with a different CID, any of Gas values, and
+	// different signature, but with all other parameters matching (source/destination,
+	// nonce, params, etc.)
+	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error) //perm:read
 	// StateGetActor returns the indicated actor's nonce and balance.
-	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
+	StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) //perm:read
 	// StateReadState returns the indicated actor's state.
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error)
+	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
 	// StateListMessages looks back and returns all messages with a matching to or from address, stopping at the given height.
-	StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error)
+	StateListMessages(ctx context.Context, match *MessageMatch, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) //perm:read
 	// StateDecodeParams attempts to decode the provided params, based on the recipient actor address and method number.
-	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error)
+	StateDecodeParams(ctx context.Context, toAddr address.Address, method abi.MethodNum, params []byte, tsk types.TipSetKey) (interface{}, error) //perm:read
 
 	// StateNetworkName returns the name of the network the node is synced to
-	StateNetworkName(context.Context) (dtypes.NetworkName, error)
+	StateNetworkName(context.Context) (dtypes.NetworkName, error) //perm:read
 	// StateMinerSectors returns info about the given miner's sectors. If the filter bitfield is nil, all sectors are included.
-	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error)
+	StateMinerSectors(context.Context, address.Address, *bitfield.BitField, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
 	// StateMinerActiveSectors returns info about sectors that a given miner is actively proving.
-	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error)
+	StateMinerActiveSectors(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) //perm:read
 	// StateMinerProvingDeadline calculates the deadline at some epoch for a proving period
 	// and returns the deadline-related calculations.
-	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)
+	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) //perm:read
 	// StateMinerPower returns the power of the indicated miner
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
+	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error) //perm:read
 	// StateMinerInfo returns info about the indicated miner
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) //perm:read
 	// StateMinerDeadlines returns all the proving deadlines for the given miner
-	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error)
+	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]Deadline, error) //perm:read
 	// StateMinerPartitions returns all partitions in the specified deadline
-	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]Partition, error)
+	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]Partition, error) //perm:read
 	// StateMinerFaults returns a bitfield indicating the faulty sectors of the given miner
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
+	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
 	// StateAllMinerFaults returns all non-expired Faults that occur within lookback epochs of the given tipset
-	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*Fault, error)
+	StateAllMinerFaults(ctx context.Context, lookback abi.ChainEpoch, ts types.TipSetKey) ([]*Fault, error) //perm:read
 	// StateMinerRecoveries returns a bitfield indicating the recovering sectors of the given miner
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
+	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) //perm:read
 	// StateMinerInitialPledgeCollateral returns the precommit deposit for the specified miner's sector
-	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
+	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerInitialPledgeCollateral returns the initial pledge collateral for the specified miner's sector
-	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error)
+	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerAvailableBalance returns the portion of a miner's balance that can be withdrawn or spent
-	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
+	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
 	// StateMinerSectorAllocated checks if a sector is allocated
-	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error)
+	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) //perm:read
 	// StateSectorPreCommitInfo returns the PreCommit info for the specified miner's sector
-	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
+	StateSectorPreCommitInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) //perm:read
 	// StateSectorGetInfo returns the on-chain info for the specified miner's sector. Returns null in case the sector info isn't found
 	// NOTE: returned info.Expiration may not be accurate in some cases, use StateSectorExpiration to get accurate
 	// expiration epoch
-	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
+	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) //perm:read
 	// StateSectorExpiration returns epoch at which given sector will expire
-	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error)
+	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
 	// StateSectorPartition finds deadline/partition with the specified sector
-	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error)
+	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
 	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
-	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
+	//
+	// NOTE: If a replacing message is found on chain, this method will return
+	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+	// result of the execution of the replacing message.
+	//
+	// If the caller wants to ensure that exactly the requested message was executed,
+	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+	// Without this check both the requested and original message may appear as
+	// successfully executed on-chain, which may look like a double-spend.
+	//
+	// A replacing message is a message with a different CID, any of Gas values, and
+	// different signature, but with all other parameters matching (source/destination,
+	// nonce, params, etc.)
+	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) //perm:read
 	// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
-	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*MsgLookup, error)
+	//
+	// NOTE: If a replacing message is found on chain, this method will return
+	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+	// result of the execution of the replacing message.
+	//
+	// If the caller wants to ensure that exactly the requested message was executed,
+	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+	// Without this check both the requested and original message may appear as
+	// successfully executed on-chain, which may look like a double-spend.
+	//
+	// A replacing message is a message with a different CID, any of Gas values, and
+	// different signature, but with all other parameters matching (source/destination,
+	// nonce, params, etc.)
+	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*MsgLookup, error) //perm:read
 	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
 	// message arrives on chain, and gets to the indicated confidence depth.
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error)
+	//
+	// NOTE: If a replacing message is found on chain, this method will return
+	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+	// result of the execution of the replacing message.
+	//
+	// If the caller wants to ensure that exactly the requested message was executed,
+	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+	// Without this check both the requested and original message may appear as
+	// successfully executed on-chain, which may look like a double-spend.
+	//
+	// A replacing message is a message with a different CID, any of Gas values, and
+	// different signature, but with all other parameters matching (source/destination,
+	// nonce, params, etc.)
+	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) //perm:read
 	// StateWaitMsgLimited looks back up to limit epochs in the chain for a message.
 	// If not found, it blocks until the message arrives on chain, and gets to the
 	// indicated confidence depth.
-	StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error)
+	//
+	// NOTE: If a replacing message is found on chain, this method will return
+	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+	// result of the execution of the replacing message.
+	//
+	// If the caller wants to ensure that exactly the requested message was executed,
+	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+	// Without this check both the requested and original message may appear as
+	// successfully executed on-chain, which may look like a double-spend.
+	//
+	// A replacing message is a message with a different CID, any of Gas values, and
+	// different signature, but with all other parameters matching (source/destination,
+	// nonce, params, etc.)
+	StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error) //perm:read
 	// StateListMiners returns the addresses of every miner that has claimed power in the Power Actor
-	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error)
+	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
 	// StateListActors returns the addresses of every actor in the state
-	StateListActors(context.Context, types.TipSetKey) ([]address.Address, error)
+	StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
 	// StateMarketBalance looks up the Escrow and Locked balances of the given address in the Storage Market
-	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (MarketBalance, error)
+	StateMarketBalance(context.Context, address.Address, types.TipSetKey) (MarketBalance, error) //perm:read
 	// StateMarketParticipants returns the Escrow and Locked balances of every participant in the Storage Market
-	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error)
+	StateMarketParticipants(context.Context, types.TipSetKey) (map[string]MarketBalance, error) //perm:read
 	// StateMarketDeals returns information about every deal in the Storage Market
-	StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error)
+	StateMarketDeals(context.Context, types.TipSetKey) (map[string]MarketDeal, error) //perm:read
 	// StateMarketStorageDeal returns information about the indicated deal
-	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error)
+	StateMarketStorageDeal(context.Context, abi.DealID, types.TipSetKey) (*MarketDeal, error) //perm:read
 	// StateLookupID retrieves the ID address of the given address
-	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error)
+	StateLookupID(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateAccountKey returns the public key address of the given ID address
-	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error)
+	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error) //perm:read
 	// StateChangedActors returns all the actors whose states change between the two given state CIDs
 	// TODO: Should this take tipset keys instead?
-	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)
+	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
-	// StateGetReceipt returns the message receipt for the given message
+	// StateGetReceipt returns the message receipt for the given message or for a
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
+	// matching gas-repriced replacing message
+	//
+	// NOTE: If the requested message was replaced, this method will return the receipt
+	// for the replacing message - if the caller needs the receipt for exactly the
+	// requested message, use StateSearchMsg().Receipt, and check that MsgLookup.Message
+	// is matching the requested CID
+	//
+	// DEPRECATED: Use StateSearchMsg, this method won't be supported in v1 API
+	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) //perm:read
 	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
-	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error)
+	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
 	// StateCompute is a flexible command that applies the given messages on the given tipset.
 	// The messages are run as though the VM were at the provided height.
-	StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error)
+	//
+	// When called, StateCompute will:
+	// - Load the provided tipset, or use the current chain head if not provided
+	// - Compute the tipset state of the provided tipset on top of the parent state
+	//   - (note that this step runs before vmheight is applied to the execution)
+	//   - Execute state upgrade if any were scheduled at the epoch, or in null
+	//     blocks preceding the tipset
+	//   - Call the cron actor on null blocks preceding the tipset
+	//   - For each block in the tipset
+	//     - Apply messages in blocks in the specified
+	//     - Award block reward by calling the reward actor
+	//   - Call the cron actor for the current epoch
+	// - If the specified vmheight is higher than the current epoch, apply any
+	//   needed state upgrades to the state
+	// - Apply the specified messages to the state
+	//
+	// The vmheight parameter sets VM execution epoch, and can be used to simulate
+	// message execution in different network versions. If the specified vmheight
+	// epoch is higher than the epoch of the specified tipset, any state upgrades
+	// until the vmheight will be executed on the state before applying messages
+	// specified by the user.
+	//
+	// Note that the initial tipset state computation is not affected by the
+	// vmheight parameter - only the messages in the `apply` set are
+	//
+	// If the caller wants to simply compute the state, vmheight should be set to
+	// the epoch of the specified tipset.
+	//
+	// Messages in the `apply` parameter must have the correct nonces, and gas
+	// values set.
+	StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error) //perm:read
 	// StateVerifierStatus returns the data cap for the given address.
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
-	StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
+	StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
 	// StateVerifiedClientStatus returns the data cap for the given address.
 	// Returns nil if there is no entry in the data cap table for the
 	// address.
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
+	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) //perm:read
 	// StateVerifiedClientStatus returns the address of the Verified Registry's root key
-	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error)
+	StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) //perm:read
 	// StateDealProviderCollateralBounds returns the min and max collateral a storage provider
 	// can issue. It takes the deal size and verified status as parameters.
-	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error)
+	StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) //perm:read
 
 	// StateCirculatingSupply returns the exact circulating supply of Filecoin at the given tipset.
 	// This is not used anywhere in the protocol itself, and is only for external consumption.
-	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error)
+	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error) //perm:read
 	// StateVMCirculatingSupplyInternal returns an approximation of the circulating supply of Filecoin at the given tipset.
 	// This is the value reported by the runtime interface to actors code.
-	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error)
+	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (CirculatingSupply, error) //perm:read
 	// StateNetworkVersion returns the network version at the given tipset
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
+	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) //perm:read
 
 	// MethodGroup: Msig
 	// The Msig methods are used to interact with multisig wallets on the
 	// filecoin network
 
 	// MsigGetAvailableBalance returns the portion of a multisig's balance that can be withdrawn or spent
-	MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
+	MsigGetAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) //perm:read
 	// MsigGetVestingSchedule returns the vesting details of a given multisig.
-	MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error)
+	MsigGetVestingSchedule(context.Context, address.Address, types.TipSetKey) (MsigVesting, error) //perm:read
 	// MsigGetVested returns the amount of FIL that vested in a multisig in a certain period.
 	// It takes the following params: <multisig address>, <start epoch>, <end epoch>
-	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error)
+	MsigGetVested(context.Context, address.Address, types.TipSetKey, types.TipSetKey) (types.BigInt, error) //perm:read
 
 	//MsigGetPending returns pending transactions for the given multisig
 	//wallet. Once pending transactions are fully approved, they will no longer
 	//appear here.
-	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error) //perm:read
 
 	// MsigCreate creates a multisig wallet
 	// It takes the following params: <required number of senders>, <approving addresses>, <unlock duration>
 	//<initial balance>, <sender address of the create msg>, <gas price>
-	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error)
+	MsigCreate(context.Context, uint64, []address.Address, abi.ChainEpoch, types.BigInt, address.Address, types.BigInt) (cid.Cid, error) //perm:sign
 	// MsigPropose proposes a multisig message
 	// It takes the following params: <multisig address>, <recipient address>, <value to transfer>,
 	// <sender address of the propose msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)
+	MsigPropose(context.Context, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
 
 	// MsigApprove approves a previously-proposed multisig message by transaction ID
 	// It takes the following params: <multisig address>, <proposed transaction ID> <signer address>
-	MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error)
+	MsigApprove(context.Context, address.Address, uint64, address.Address) (cid.Cid, error) //perm:sign
 
 	// MsigApproveTxnHash approves a previously-proposed multisig message, specified
 	// using both transaction ID and a hash of the parameters used in the
@@ -497,80 +617,80 @@ type FullNode interface {
 	// exactly the transaction you think you are.
 	// It takes the following params: <multisig address>, <proposed message ID>, <proposer address>, <recipient address>, <value to transfer>,
 	// <sender address of the approve msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)
+	MsigApproveTxnHash(context.Context, address.Address, uint64, address.Address, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
 
 	// MsigCancel cancels a previously-proposed multisig message
 	// It takes the following params: <multisig address>, <proposed transaction ID>, <recipient address>, <value to transfer>,
 	// <sender address of the cancel msg>, <method to call in the proposed message>, <params to include in the proposed message>
-	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error)
+	MsigCancel(context.Context, address.Address, uint64, address.Address, types.BigInt, address.Address, uint64, []byte) (cid.Cid, error) //perm:sign
 	// MsigAddPropose proposes adding a signer in the multisig
 	// It takes the following params: <multisig address>, <sender address of the propose msg>,
 	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddPropose(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error)
+	MsigAddPropose(context.Context, address.Address, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
 	// MsigAddApprove approves a previously proposed AddSigner message
 	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
 	// <proposer address>, <new signer>, <whether the number of required signers should be increased>
-	MsigAddApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error)
+	MsigAddApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, bool) (cid.Cid, error) //perm:sign
 	// MsigAddCancel cancels a previously proposed AddSigner message
 	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
 	// <new signer>, <whether the number of required signers should be increased>
-	MsigAddCancel(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error)
+	MsigAddCancel(context.Context, address.Address, address.Address, uint64, address.Address, bool) (cid.Cid, error) //perm:sign
 	// MsigSwapPropose proposes swapping 2 signers in the multisig
 	// It takes the following params: <multisig address>, <sender address of the propose msg>,
 	// <old signer>, <new signer>
-	MsigSwapPropose(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error)
+	MsigSwapPropose(context.Context, address.Address, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
 	// MsigSwapApprove approves a previously proposed SwapSigner
 	// It takes the following params: <multisig address>, <sender address of the approve msg>, <proposed message ID>,
 	// <proposer address>, <old signer>, <new signer>
-	MsigSwapApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error)
+	MsigSwapApprove(context.Context, address.Address, address.Address, uint64, address.Address, address.Address, address.Address) (cid.Cid, error) //perm:sign
 	// MsigSwapCancel cancels a previously proposed SwapSigner message
 	// It takes the following params: <multisig address>, <sender address of the cancel msg>, <proposed message ID>,
 	// <old signer>, <new signer>
-	MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error)
+	MsigSwapCancel(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error) //perm:sign
 
 	// MsigRemoveSigner proposes the removal of a signer from the multisig.
 	// It accepts the multisig to make the change on, the proposer address to
 	// send the message from, the address to be removed, and a boolean
 	// indicating whether or not the signing threshold should be lowered by one
 	// along with the address removal.
-	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error)
+	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error) //perm:sign
 
 	// MarketAddBalance adds funds to the market actor
-	MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error)
+	MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
 	// MarketGetReserved gets the amount of funds that are currently reserved for the address
-	MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error)
+	MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error) //perm:sign
 	// MarketReserveFunds reserves funds for a deal
-	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error)
+	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
 	// MarketReleaseFunds releases funds reserved by MarketReserveFunds
-	MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error
+	MarketReleaseFunds(ctx context.Context, addr address.Address, amt types.BigInt) error //perm:sign
 	// MarketWithdraw withdraws unlocked funds from the market actor
-	MarketWithdraw(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error)
+	MarketWithdraw(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) //perm:sign
 
 	// MethodGroup: Paych
 	// The Paych methods are for interacting with and managing payment channels
 
-	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)
+	PaychGet(ctx context.Context, from, to address.Address, amt types.BigInt) (*ChannelInfo, error)                     //perm:sign
-	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)
+	PaychGetWaitReady(context.Context, cid.Cid) (address.Address, error)                                                //perm:sign
-	PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error)
+	PaychAvailableFunds(ctx context.Context, ch address.Address) (*ChannelAvailableFunds, error)                        //perm:sign
-	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error)
+	PaychAvailableFundsByFromTo(ctx context.Context, from, to address.Address) (*ChannelAvailableFunds, error)          //perm:sign
-	PaychList(context.Context) ([]address.Address, error)
+	PaychList(context.Context) ([]address.Address, error)                                                               //perm:read
-	PaychStatus(context.Context, address.Address) (*PaychStatus, error)
+	PaychStatus(context.Context, address.Address) (*PaychStatus, error)                                                 //perm:read
-	PaychSettle(context.Context, address.Address) (cid.Cid, error)
+	PaychSettle(context.Context, address.Address) (cid.Cid, error)                                                      //perm:sign
-	PaychCollect(context.Context, address.Address) (cid.Cid, error)
+	PaychCollect(context.Context, address.Address) (cid.Cid, error)                                                     //perm:sign
-	PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)
+	PaychAllocateLane(ctx context.Context, ch address.Address) (uint64, error)                                          //perm:sign
-	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)
+	PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)        //perm:sign
-	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error
+	PaychVoucherCheckValid(context.Context, address.Address, *paych.SignedVoucher) error                                //perm:read
-	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)
+	PaychVoucherCheckSpendable(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (bool, error)    //perm:read
-	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*VoucherCreateResult, error)
+	PaychVoucherCreate(context.Context, address.Address, types.BigInt, uint64) (*VoucherCreateResult, error)            //perm:sign
-	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
+	PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) //perm:write
-	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)
+	PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error)                                  //perm:write
-	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)
+	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error)         //perm:sign
 
 	// CreateBackup creates node backup onder the specified file name. The
 	// method requires that the lotus daemon is running with the
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
 	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error
+	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 }
 
 type FileRef struct {

api/api_gateway.go

@@ -8,13 +8,13 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
 
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-type GatewayAPI interface {
+type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainHead(ctx context.Context) (*types.TipSet, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
@@ -39,7 +39,7 @@ type GatewayAPI interface {
 	StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error)
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
+	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
 	StateSearchMsg(ctx context.Context, msg cid.Cid) (*MsgLookup, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)

api/api_storage.go

@@ -28,124 +28,148 @@ import (
 type StorageMiner interface {
 	Common
 
-	ActorAddress(context.Context) (address.Address, error)
+	ActorAddress(context.Context) (address.Address, error) //perm:read
 
-	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error)
+	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
-	ActorAddressConfig(ctx context.Context) (AddressConfig, error)
+	ActorAddressConfig(ctx context.Context) (AddressConfig, error)            //perm:read
 
-	MiningBase(context.Context) (*types.TipSet, error)
+	MiningBase(context.Context) (*types.TipSet, error) //perm:read
 
 	// Temp api for testing
-	PledgeSector(context.Context) (abi.SectorID, error)
+	PledgeSector(context.Context) (abi.SectorID, error) //perm:write
 
 	// Get the status of a given sector by ID
-	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error)
+	SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read
 
 	// List all staged sectors
-	SectorsList(context.Context) ([]abi.SectorNumber, error)
+	SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read
 
 	// Get summary info of sectors
-	SectorsSummary(ctx context.Context) (map[SectorState]int, error)
+	SectorsSummary(ctx context.Context) (map[SectorState]int, error) //perm:read
 
 	// List sectors in particular states
-	SectorsListInStates(context.Context, []SectorState) ([]abi.SectorNumber, error)
+	SectorsListInStates(context.Context, []SectorState) ([]abi.SectorNumber, error) //perm:read
 
-	SectorsRefs(context.Context) (map[string][]SealedRef, error)
+	SectorsRefs(context.Context) (map[string][]SealedRef, error) //perm:read
 
 	// SectorStartSealing can be called on sectors in Empty or WaitDeals states
 	// to trigger sealing early
-	SectorStartSealing(context.Context, abi.SectorNumber) error
+	SectorStartSealing(context.Context, abi.SectorNumber) error //perm:write
 	// SectorSetSealDelay sets the time that a newly-created sector
 	// waits for more deals before it starts sealing
-	SectorSetSealDelay(context.Context, time.Duration) error
+	SectorSetSealDelay(context.Context, time.Duration) error //perm:write
 	// SectorGetSealDelay gets the time that a newly-created sector
 	// waits for more deals before it starts sealing
-	SectorGetSealDelay(context.Context) (time.Duration, error)
+	SectorGetSealDelay(context.Context) (time.Duration, error) //perm:read
 	// SectorSetExpectedSealDuration sets the expected time for a sector to seal
-	SectorSetExpectedSealDuration(context.Context, time.Duration) error
+	SectorSetExpectedSealDuration(context.Context, time.Duration) error //perm:write
 	// SectorGetExpectedSealDuration gets the expected time for a sector to seal
-	SectorGetExpectedSealDuration(context.Context) (time.Duration, error)
+	SectorGetExpectedSealDuration(context.Context) (time.Duration, error) //perm:read
-	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
+	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error   //perm:admin
 	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
 	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
-	SectorRemove(context.Context, abi.SectorNumber) error
+	SectorRemove(context.Context, abi.SectorNumber) error //perm:admin
 	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
 	// automatically removes it from storage
-	SectorTerminate(context.Context, abi.SectorNumber) error
+	SectorTerminate(context.Context, abi.SectorNumber) error //perm:admin
 	// SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
 	// Returns null if message wasn't sent
-	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error)
+	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) //perm:admin
 	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
-	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)
+	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)  //perm:admin
-	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error
+	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error //perm:admin
-
-	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
-	StorageLocal(ctx context.Context) (map[stores.ID]string, error)
-	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error)
 
 	// WorkerConnect tells the node to connect to workers RPC
-	WorkerConnect(context.Context, string) error
+	WorkerConnect(context.Context, string) error                              //perm:admin retry:true
-	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error)
+	WorkerStats(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) //perm:admin
-	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)
+	WorkerJobs(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error)  //perm:admin
-	storiface.WorkerReturn
+
+	//storiface.WorkerReturn
+	ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, err *storiface.CallError) error                //perm:admin retry:true
+	ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error //perm:admin retry:true
+	ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, err *storiface.CallError) error //perm:admin retry:true
+	ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, err *storiface.CallError) error       //perm:admin retry:true
+	ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, err *storiface.CallError) error          //perm:admin retry:true
+	ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                            //perm:admin retry:true
+	ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                           //perm:admin retry:true
+	ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
+	ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                               //perm:admin retry:true
+	ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, err *storiface.CallError) error                        //perm:admin retry:true
+	ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error                                     //perm:admin retry:true
 
 	// SealingSchedDiag dumps internal sealing scheduler state
-	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error)
+	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
-	SealingAbort(ctx context.Context, call storiface.CallID) error
+	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
 
-	stores.SectorIndex
+	//stores.SectorIndex
+	StorageAttach(context.Context, stores.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
+	StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error)                                                                                                 //perm:admin
+	StorageReportHealth(context.Context, stores.ID, stores.HealthReport) error                                                                                          //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
+	StorageDropSector(ctx context.Context, storageID stores.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]stores.SectorStorageInfo, error) //perm:admin
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]stores.StorageInfo, error)           //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                          //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                               //perm:admin
 
-	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
+	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) //perm:admin
-	MarketListDeals(ctx context.Context) ([]MarketDeal, error)
+	StorageLocal(ctx context.Context) (map[stores.ID]string, error)       //perm:admin
-	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
+	StorageStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) //perm:admin
-	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)
+
-	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
+	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write
-	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
+	MarketListDeals(ctx context.Context) ([]MarketDeal, error)                                                                                                                           //perm:read
-	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)
+	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                           //perm:read
-	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error
+	MarketGetDealUpdates(ctx context.Context) (<-chan storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)
+	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                    //perm:read
-	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)
+	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
-	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)
+	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           //perm:read
+	MarketSetRetrievalAsk(ctx context.Context, rask *retrievalmarket.Ask) error                                                                                                          //perm:admin
+	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
+	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
+	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
 	// MarketRestartDataTransfer attempts to restart a data transfer with the given transfer ID and other peer
-	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	// MarketCancelDataTransfer cancels a data transfer with the given transfer ID and other peer
-	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error
+	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
-	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)
+	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
-	MarketPublishPendingDeals(ctx context.Context) error
+	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
 
-	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
+	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error //perm:admin
-	DealsList(ctx context.Context) ([]MarketDeal, error)
+	DealsList(ctx context.Context) ([]MarketDeal, error)                         //perm:admin
-	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)
+	DealsConsiderOnlineStorageDeals(context.Context) (bool, error)               //perm:admin
-	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error
+	DealsSetConsiderOnlineStorageDeals(context.Context, bool) error              //perm:admin
-	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)
+	DealsConsiderOnlineRetrievalDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error
+	DealsSetConsiderOnlineRetrievalDeals(context.Context, bool) error            //perm:admin
-	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)
+	DealsPieceCidBlocklist(context.Context) ([]cid.Cid, error)                   //perm:admin
-	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error
+	DealsSetPieceCidBlocklist(context.Context, []cid.Cid) error                  //perm:admin
-	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)
+	DealsConsiderOfflineStorageDeals(context.Context) (bool, error)              //perm:admin
-	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error
+	DealsSetConsiderOfflineStorageDeals(context.Context, bool) error             //perm:admin
-	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)
+	DealsConsiderOfflineRetrievalDeals(context.Context) (bool, error)            //perm:admin
-	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error
+	DealsSetConsiderOfflineRetrievalDeals(context.Context, bool) error           //perm:admin
-	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)
+	DealsConsiderVerifiedStorageDeals(context.Context) (bool, error)             //perm:admin
-	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error
+	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error            //perm:admin
-	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)
+	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)           //perm:admin
-	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error
+	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error          //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error
+	StorageAddLocal(ctx context.Context, path string) error //perm:admin
 
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)
+	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)
+	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error)
+	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)
+	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)   //perm:read
 
 	// CreateBackup creates node backup onder the specified file name. The
 	// method requires that the lotus-miner is running with the
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
 	// the path specified when calling CreateBackup is within the base path
-	CreateBackup(ctx context.Context, fpath string) error
+	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 }
 
+var _ storiface.WorkerReturn = *new(StorageMiner)
+var _ stores.SectorIndex = *new(StorageMiner)
+
 type SealRes struct {
 	Err   string
 	GoErr error `json:"-"`

api/api_test.go

@@ -37,6 +37,18 @@ func TestDoesntDependOnFFI(t *testing.T) {
 	}
 }
 
+func TestDoesntDependOnBuild(t *testing.T) {
+	deps, err := exec.Command(goCmd(), "list", "-deps", "github.com/filecoin-project/lotus/api").Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range strings.Fields(string(deps)) {
+		if pkg == "github.com/filecoin-project/build" {
+			t.Fatal("api depends on filecoin-ffi")
+		}
+	}
+}
+
 func TestReturnTypes(t *testing.T) {
 	errType := reflect.TypeOf(new(error)).Elem()
 	bareIface := reflect.TypeOf(new(interface{})).Elem()
@@ -99,5 +111,5 @@ func TestReturnTypes(t *testing.T) {
 	t.Run("common", tst(new(Common)))
 	t.Run("full", tst(new(FullNode)))
 	t.Run("miner", tst(new(StorageMiner)))
-	t.Run("worker", tst(new(WorkerAPI)))
+	t.Run("worker", tst(new(Worker)))
 }

api/api_wallet.go

@@ -34,7 +34,7 @@ type MsgMeta struct {
 	Extra []byte
 }
 
-type WalletAPI interface {
+type Wallet interface {
 	WalletNew(context.Context, types.KeyType) (address.Address, error)
 	WalletHas(context.Context, address.Address) (bool, error)
 	WalletList(context.Context) ([]address.Address, error)

api/api_worker.go

@@ -2,46 +2,62 @@ package api
 
 import (
 	"context"
+	"io"
 
 	"github.com/google/uuid"
+	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/specs-storage/storage"
 )
 
-type WorkerAPI interface {
+type Worker interface {
-	Version(context.Context) (Version, error)
+	Version(context.Context) (Version, error) //perm:admin
-	// TODO: Info() (name, ...) ?
 
-	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight
+	// TaskType -> Weight
-	Paths(context.Context) ([]stores.StoragePath, error)
+	TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) //perm:admin
-	Info(context.Context) (storiface.WorkerInfo, error)
+	Paths(context.Context) ([]stores.StoragePath, error)                //perm:admin
+	Info(context.Context) (storiface.WorkerInfo, error)                 //perm:admin
 
-	storiface.WorkerCalls
+	// storiface.WorkerCalls
+	AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (storiface.CallID, error)                    //perm:admin
+	SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storiface.CallID, error)                                                           //perm:admin
+	SealPreCommit2(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (storiface.CallID, error)                                                                                  //perm:admin
+	SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error) //perm:admin
+	SealCommit2(ctx context.Context, sector storage.SectorRef, c1o storage.Commit1Out) (storiface.CallID, error)                                                                                         //perm:admin
+	FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)                                                                                //perm:admin
+	ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)                                                                                 //perm:admin
+	MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)                                                                                 //perm:admin
+	UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error)                                           //perm:admin
+	ReadPiece(context.Context, io.Writer, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (storiface.CallID, error)                                                               //perm:admin
+	Fetch(context.Context, storage.SectorRef, storiface.SectorFileType, storiface.PathType, storiface.AcquireMode) (storiface.CallID, error)                                                             //perm:admin
 
-	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error
+	TaskDisable(ctx context.Context, tt sealtasks.TaskType) error //perm:admin
-	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error
+	TaskEnable(ctx context.Context, tt sealtasks.TaskType) error  //perm:admin
 
 	// Storage / Other
-	Remove(ctx context.Context, sector abi.SectorID) error
+	Remove(ctx context.Context, sector abi.SectorID) error //perm:admin
 
-	StorageAddLocal(ctx context.Context, path string) error
+	StorageAddLocal(ctx context.Context, path string) error //perm:admin
 
 	// SetEnabled marks the worker as enabled/disabled. Not that this setting
 	// may take a few seconds to propagate to task scheduler
-	SetEnabled(ctx context.Context, enabled bool) error
+	SetEnabled(ctx context.Context, enabled bool) error //perm:admin
 
-	Enabled(ctx context.Context) (bool, error)
+	Enabled(ctx context.Context) (bool, error) //perm:admin
 
 	// WaitQuiet blocks until there are no tasks running
-	WaitQuiet(ctx context.Context) error
+	WaitQuiet(ctx context.Context) error //perm:admin
 
 	// returns a random UUID of worker session, generated randomly when worker
 	// process starts
-	ProcessSession(context.Context) (uuid.UUID, error)
+	ProcessSession(context.Context) (uuid.UUID, error) //perm:admin
 
 	// Like ProcessSession, but returns an error when worker is disabled
-	Session(context.Context) (uuid.UUID, error)
+	Session(context.Context) (uuid.UUID, error) //perm:admin
 }
+
+var _ storiface.WorkerCalls = *new(Worker)

api/apistruct/permissioned.go

@@ -31,13 +31,13 @@ func PermissionedFullAPI(a api.FullNode) api.FullNode {
 	return &out
 }
 
-func PermissionedWorkerAPI(a api.WorkerAPI) api.WorkerAPI {
+func PermissionedWorkerAPI(a api.Worker) api.Worker {
 	var out WorkerStruct
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	return &out
 }
 
-func PermissionedWalletAPI(a api.WalletAPI) api.WalletAPI {
+func PermissionedWalletAPI(a api.Wallet) api.Wallet {
 	var out WalletStruct
 	auth.PermissionedProxy(AllPermissions, DefaultPerms, a, &out.Internal)
 	return &out

api/client/client.go

@@ -54,7 +54,7 @@ func NewStorageMinerRPC(ctx context.Context, addr string, requestHeader http.Hea
 	return &res, closer, err
 }
 
-func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Worker, jsonrpc.ClientCloser, error) {
 	u, err := url.Parse(addr)
 	if err != nil {
 		return nil, nil, err
@@ -84,7 +84,7 @@ func NewWorkerRPC(ctx context.Context, addr string, requestHeader http.Header) (
 }
 
 // NewGatewayRPC creates a new http jsonrpc client for a gateway node.
-func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
+func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header, opts ...jsonrpc.Option) (api.Gateway, jsonrpc.ClientCloser, error) {
 	var res apistruct.GatewayStruct
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{
@@ -97,7 +97,7 @@ func NewGatewayRPC(ctx context.Context, addr string, requestHeader http.Header,
 	return &res, closer, err
 }
 
-func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.WalletAPI, jsonrpc.ClientCloser, error) {
+func NewWalletRPC(ctx context.Context, addr string, requestHeader http.Header) (api.Wallet, jsonrpc.ClientCloser, error) {
 	var res apistruct.WalletStruct
 	closer, err := jsonrpc.NewMergeClient(ctx, addr, "Filecoin",
 		[]interface{}{

api/docgen-openrpc/cmd/docgen_openrpc.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+
+	"github.com/filecoin-project/lotus/api/apistruct"
+	docgen_openrpc "github.com/filecoin-project/lotus/api/docgen-openrpc"
+)
+
+/*
+main defines a small program that writes an OpenRPC document describing
+a Lotus API to stdout.
+
+If the first argument is "miner", the document will describe the StorageMiner API.
+If not (no, or any other args), the document will describe the Full API.
+
+Use:
+
+		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"]
+
+	With gzip compression: a '-gzip' flag is made available as an optional third argument. Note that position matters.
+
+		go run ./api/openrpc/cmd ["api/api_full.go"|"api/api_storage.go"|"api/api_worker.go"] ["FullNode"|"StorageMiner"|"Worker"] -gzip
+
+*/
+
+func main() {
+	doc := docgen_openrpc.NewLotusOpenRPCDocument()
+
+	switch os.Args[2] {
+	case "FullNode":
+		doc.RegisterReceiverName("Filecoin", &apistruct.FullNodeStruct{})
+	case "StorageMiner":
+		doc.RegisterReceiverName("Filecoin", &apistruct.StorageMinerStruct{})
+	case "Worker":
+		doc.RegisterReceiverName("Filecoin", &apistruct.WorkerStruct{})
+	}
+
+	out, err := doc.Discover()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	var jsonOut []byte
+	var writer io.WriteCloser
+
+	// Use os.Args to handle a somewhat hacky flag for the gzip option.
+	// Could use flags package to handle this more cleanly, but that requires changes elsewhere
+	// the scope of which just isn't warranted by this one use case which will usually be run
+	// programmatically anyways.
+	if len(os.Args) > 3 && os.Args[3] == "-gzip" {
+		jsonOut, err = json.Marshal(out)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		writer = gzip.NewWriter(os.Stdout)
+	} else {
+		jsonOut, err = json.MarshalIndent(out, "", "    ")
+		if err != nil {
+			log.Fatalln(err)
+		}
+		writer = os.Stdout
+	}
+
+	_, err = writer.Write(jsonOut)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

api/docgen-openrpc/openrpc.go

@@ -0,0 +1,172 @@
+package docgenopenrpc
+
+import (
+	"encoding/json"
+	"go/ast"
+	"net"
+	"os"
+	"reflect"
+
+	"github.com/alecthomas/jsonschema"
+	go_openrpc_reflect "github.com/etclabscore/go-openrpc-reflect"
+	"github.com/filecoin-project/lotus/api/docgen"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/ipfs/go-cid"
+	meta_schema "github.com/open-rpc/meta-schema"
+)
+
+// Comments holds API method comments collected by AST parsing.
+var Comments map[string]string
+
+// GroupDocs holds documentation for documentation groups.
+var GroupDocs map[string]string
+
+func init() {
+	Comments, GroupDocs = docgen.ParseApiASTInfo(os.Args[1], os.Args[2])
+}
+
+// schemaDictEntry represents a type association passed to the jsonschema reflector.
+type schemaDictEntry struct {
+	example interface{}
+	rawJson string
+}
+
+const integerD = `{
+          "title": "number",
+          "type": "number",
+          "description": "Number is a number"
+        }`
+
+const cidCidD = `{"title": "Content Identifier", "type": "string", "description": "Cid represents a self-describing content addressed identifier. It is formed by a Version, a Codec (which indicates a multicodec-packed content type) and a Multihash."}`
+
+func OpenRPCSchemaTypeMapper(ty reflect.Type) *jsonschema.Type {
+	unmarshalJSONToJSONSchemaType := func(input string) *jsonschema.Type {
+		var js jsonschema.Type
+		err := json.Unmarshal([]byte(input), &js)
+		if err != nil {
+			panic(err)
+		}
+		return &js
+	}
+
+	if ty.Kind() == reflect.Ptr {
+		ty = ty.Elem()
+	}
+
+	if ty == reflect.TypeOf((*interface{})(nil)).Elem() {
+		return &jsonschema.Type{Type: "object", AdditionalProperties: []byte("true")}
+	}
+
+	// Second, handle other types.
+	// Use a slice instead of a map because it preserves order, as a logic safeguard/fallback.
+	dict := []schemaDictEntry{
+		{cid.Cid{}, cidCidD},
+	}
+
+	for _, d := range dict {
+		if reflect.TypeOf(d.example) == ty {
+			tt := unmarshalJSONToJSONSchemaType(d.rawJson)
+
+			return tt
+		}
+	}
+
+	// Handle primitive types in case there are generic cases
+	// specific to our services.
+	switch ty.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		// Return all integer types as the hex representation integer schemea.
+		ret := unmarshalJSONToJSONSchemaType(integerD)
+		return ret
+	case reflect.Uintptr:
+		return &jsonschema.Type{Type: "number", Title: "uintptr-title"}
+	case reflect.Struct:
+	case reflect.Map:
+	case reflect.Slice, reflect.Array:
+	case reflect.Float32, reflect.Float64:
+	case reflect.Bool:
+	case reflect.String:
+	case reflect.Ptr, reflect.Interface:
+	default:
+	}
+
+	return nil
+}
+
+// NewLotusOpenRPCDocument defines application-specific documentation and configuration for its OpenRPC document.
+func NewLotusOpenRPCDocument() *go_openrpc_reflect.Document {
+	d := &go_openrpc_reflect.Document{}
+
+	// Register "Meta" document fields.
+	// These include getters for
+	// - Servers object
+	// - Info object
+	// - ExternalDocs object
+	//
+	// These objects represent server-specific data that cannot be
+	// reflected.
+	d.WithMeta(&go_openrpc_reflect.MetaT{
+		GetServersFn: func() func(listeners []net.Listener) (*meta_schema.Servers, error) {
+			return func(listeners []net.Listener) (*meta_schema.Servers, error) {
+				return nil, nil
+			}
+		},
+		GetInfoFn: func() (info *meta_schema.InfoObject) {
+			info = &meta_schema.InfoObject{}
+			title := "Lotus RPC API"
+			info.Title = (*meta_schema.InfoObjectProperties)(&title)
+
+			version := build.BuildVersion
+			info.Version = (*meta_schema.InfoObjectVersion)(&version)
+			return info
+		},
+		GetExternalDocsFn: func() (exdocs *meta_schema.ExternalDocumentationObject) {
+			return nil // FIXME
+		},
+	})
+
+	// Use a provided Ethereum default configuration as a base.
+	appReflector := &go_openrpc_reflect.EthereumReflectorT{}
+
+	// Install overrides for the json schema->type map fn used by the jsonschema reflect package.
+	appReflector.FnSchemaTypeMap = func() func(ty reflect.Type) *jsonschema.Type {
+		return OpenRPCSchemaTypeMapper
+	}
+
+	appReflector.FnIsMethodEligible = func(m reflect.Method) bool {
+		for i := 0; i < m.Func.Type().NumOut(); i++ {
+			if m.Func.Type().Out(i).Kind() == reflect.Chan {
+				return false
+			}
+		}
+		return go_openrpc_reflect.EthereumReflector.IsMethodEligible(m)
+	}
+	appReflector.FnGetMethodName = func(moduleName string, r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
+		if m.Name == "ID" {
+			return moduleName + "_ID", nil
+		}
+		if moduleName == "rpc" && m.Name == "Discover" {
+			return "rpc.discover", nil
+		}
+
+		return moduleName + "." + m.Name, nil
+	}
+
+	appReflector.FnGetMethodSummary = func(r reflect.Value, m reflect.Method, funcDecl *ast.FuncDecl) (string, error) {
+		if v, ok := Comments[m.Name]; ok {
+			return v, nil
+		}
+		return "", nil // noComment
+	}
+
+	appReflector.FnSchemaExamples = func(ty reflect.Type) (examples *meta_schema.Examples, err error) {
+		v := docgen.ExampleValue("unknown", ty, ty) // This isn't ideal, but seems to work well enough.
+		return &meta_schema.Examples{
+			meta_schema.AlwaysTrue(v),
+		}, nil
+	}
+
+	// Finally, register the configured reflector to the document.
+	d.WithReflector(appReflector)
+	return d
+}

api/docgen/cmd/docgen.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/apistruct"
+	"github.com/filecoin-project/lotus/api/docgen"
+)
+
+func main() {
+	comments, groupComments := docgen.ParseApiASTInfo(os.Args[1], os.Args[2])
+
+	groups := make(map[string]*docgen.MethodGroup)
+
+	var t reflect.Type
+	var permStruct, commonPermStruct reflect.Type
+
+	switch os.Args[2] {
+	case "FullNode":
+		t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
+		permStruct = reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
+		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
+	case "StorageMiner":
+		t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
+		permStruct = reflect.TypeOf(apistruct.StorageMinerStruct{}.Internal)
+		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
+	case "Worker":
+		t = reflect.TypeOf(new(struct{ api.Worker })).Elem()
+		permStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
+		commonPermStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
+	default:
+		panic("unknown type")
+	}
+
+	for i := 0; i < t.NumMethod(); i++ {
+		m := t.Method(i)
+
+		groupName := docgen.MethodGroupFromName(m.Name)
+
+		g, ok := groups[groupName]
+		if !ok {
+			g = new(docgen.MethodGroup)
+			g.Header = groupComments[groupName]
+			g.GroupName = groupName
+			groups[groupName] = g
+		}
+
+		var args []interface{}
+		ft := m.Func.Type()
+		for j := 2; j < ft.NumIn(); j++ {
+			inp := ft.In(j)
+			args = append(args, docgen.ExampleValue(m.Name, inp, nil))
+		}
+
+		v, err := json.MarshalIndent(args, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+
+		outv := docgen.ExampleValue(m.Name, ft.Out(0), nil)
+
+		ov, err := json.MarshalIndent(outv, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+
+		g.Methods = append(g.Methods, &docgen.Method{
+			Name:            m.Name,
+			Comment:         comments[m.Name],
+			InputExample:    string(v),
+			ResponseExample: string(ov),
+		})
+	}
+
+	var groupslice []*docgen.MethodGroup
+	for _, g := range groups {
+		groupslice = append(groupslice, g)
+	}
+
+	sort.Slice(groupslice, func(i, j int) bool {
+		return groupslice[i].GroupName < groupslice[j].GroupName
+	})
+
+	fmt.Printf("# Groups\n")
+
+	for _, g := range groupslice {
+		fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
+		for _, method := range g.Methods {
+			fmt.Printf("  * [%s](#%s)\n", method.Name, method.Name)
+		}
+	}
+
+	for _, g := range groupslice {
+		g := g
+		fmt.Printf("## %s\n", g.GroupName)
+		fmt.Printf("%s\n\n", g.Header)
+
+		sort.Slice(g.Methods, func(i, j int) bool {
+			return g.Methods[i].Name < g.Methods[j].Name
+		})
+
+		for _, m := range g.Methods {
+			fmt.Printf("### %s\n", m.Name)
+			fmt.Printf("%s\n\n", m.Comment)
+
+			meth, ok := permStruct.FieldByName(m.Name)
+			if !ok {
+				meth, ok = commonPermStruct.FieldByName(m.Name)
+				if !ok {
+					panic("no perms for method: " + m.Name)
+				}
+			}
+
+			perms := meth.Tag.Get("perm")
+
+			fmt.Printf("Perms: %s\n\n", perms)
+
+			if strings.Count(m.InputExample, "\n") > 0 {
+				fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
+			} else {
+				fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
+			}
+
+			if strings.Count(m.ResponseExample, "\n") > 0 {
+				fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
+			} else {
+				fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
+			}
+		}
+	}
+}

api/docgen/docgen.go

@@ -1,18 +1,18 @@
-package main
+package docgen
 
 import (
-	"encoding/json"
 	"fmt"
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"os"
+	"path/filepath"
 	"reflect"
-	"sort"
 	"strings"
 	"time"
 	"unicode"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-filestore"
@@ -23,8 +23,6 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/multiformats/go-multiaddr"
 
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	filestore2 "github.com/filecoin-project/go-fil-markets/filestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
@@ -36,7 +34,7 @@ import (
 	"github.com/filecoin-project/go-state-types/exitcode"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apistruct"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
@@ -89,6 +87,8 @@ func init() {
 	addExample(pid)
 	addExample(&pid)
 
+	multistoreIDExample := multistore.StoreID(50)
+
 	addExample(bitfield.NewFromSet([]uint64{5}))
 	addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)
 	addExample(abi.RegisteredPoStProof_StackedDrgWindow32GiBV1)
@@ -118,7 +118,8 @@ func init() {
 	addExample(time.Minute)
 	addExample(datatransfer.TransferID(3))
 	addExample(datatransfer.Ongoing)
-	addExample(multistore.StoreID(50))
+	addExample(multistoreIDExample)
+	addExample(&multistoreIDExample)
 	addExample(retrievalmarket.ClientEventDealAccepted)
 	addExample(retrievalmarket.DealStatusNew)
 	addExample(network.ReachabilityPublic)
@@ -126,17 +127,17 @@ func init() {
 	addExample(map[string]int{"name": 42})
 	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
-		Msg:    exampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
+		Msg:    ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: exampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
+		MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
 	})
 	addExample(map[string]types.Actor{
-		"t01236": exampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
+		"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),
 	})
 	addExample(map[string]api.MarketDeal{
-		"t026363": exampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
+		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketDeal{}), nil).(api.MarketDeal),
 	})
 	addExample(map[string]api.MarketBalance{
-		"t026363": exampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
+		"t026363": ExampleValue("init", reflect.TypeOf(api.MarketBalance{}), nil).(api.MarketBalance),
 	})
 	addExample(map[string]*pubsub.TopicScoreSnapshot{
 		"/blocks": {
@@ -251,9 +252,17 @@ func init() {
 		sealtasks.TTPreCommit2: {},
 	})
 	addExample(sealtasks.TTCommit2)
+	addExample(apitypes.OpenRPCDocument{
+		"openrpc": "1.2.6",
+		"info": map[string]interface{}{
+			"title":   "Lotus RPC API",
+			"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
+		},
+		"methods": []interface{}{}},
+	)
 }
 
-func exampleValue(method string, t, parent reflect.Type) interface{} {
+func ExampleValue(method string, t, parent reflect.Type) interface{} {
 	v, ok := ExampleValues[t]
 	if ok {
 		return v
@@ -262,10 +271,10 @@ func exampleValue(method string, t, parent reflect.Type) interface{} {
 	switch t.Kind() {
 	case reflect.Slice:
 		out := reflect.New(t).Elem()
-		reflect.Append(out, reflect.ValueOf(exampleValue(method, t.Elem(), t)))
+		reflect.Append(out, reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
 		return out.Interface()
 	case reflect.Chan:
-		return exampleValue(method, t.Elem(), nil)
+		return ExampleValue(method, t.Elem(), nil)
 	case reflect.Struct:
 		es := exampleStruct(method, t, parent)
 		v := reflect.ValueOf(es).Elem().Interface()
@@ -274,7 +283,7 @@ func exampleValue(method string, t, parent reflect.Type) interface{} {
 	case reflect.Array:
 		out := reflect.New(t).Elem()
 		for i := 0; i < t.Len(); i++ {
-			out.Index(i).Set(reflect.ValueOf(exampleValue(method, t.Elem(), t)))
+			out.Index(i).Set(reflect.ValueOf(ExampleValue(method, t.Elem(), t)))
 		}
 		return out.Interface()
 
@@ -299,7 +308,7 @@ func exampleStruct(method string, t, parent reflect.Type) interface{} {
 			continue
 		}
 		if strings.Title(f.Name) == f.Name {
-			ns.Elem().Field(i).Set(reflect.ValueOf(exampleValue(method, f.Type, t)))
+			ns.Elem().Field(i).Set(reflect.ValueOf(ExampleValue(method, f.Type, t)))
 		}
 	}
 
@@ -331,13 +340,24 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
 	return v
 }
 
-const noComment = "There are not yet any comments for this method."
+const NoComment = "There are not yet any comments for this method."
 
-func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]string) { //nolint:golint
+func ParseApiASTInfo(apiFile, iface string) (comments map[string]string, groupDocs map[string]string) { //nolint:golint
 	fset := token.NewFileSet()
-	pkgs, err := parser.ParseDir(fset, "./api", nil, parser.AllErrors|parser.ParseComments)
+	apiDir, err := filepath.Abs("./api")
+	if err != nil {
+		fmt.Println("./api filepath absolute error: ", err)
+		return
+	}
+	apiFile, err = filepath.Abs(apiFile)
+	if err != nil {
+		fmt.Println("filepath absolute error: ", err, "file:", apiFile)
+		return
+	}
+	pkgs, err := parser.ParseDir(fset, apiDir, nil, parser.AllErrors|parser.ParseComments)
 	if err != nil {
 		fmt.Println("parse error: ", err)
+		return
 	}
 
 	ap := pkgs["api"]
@@ -349,14 +369,14 @@ func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]strin
 	v := &Visitor{iface, make(map[string]ast.Node)}
 	ast.Walk(v, pkgs["api"])
 
-	groupDocs := make(map[string]string)
+	comments = make(map[string]string)
-	out := make(map[string]string)
+	groupDocs = make(map[string]string)
 	for mn, node := range v.Methods {
-		cs := cmap.Filter(node).Comments()
+		filteredComments := cmap.Filter(node).Comments()
-		if len(cs) == 0 {
+		if len(filteredComments) == 0 {
-			out[mn] = noComment
+			comments[mn] = NoComment
 		} else {
-			for _, c := range cs {
+			for _, c := range filteredComments {
 				if strings.HasPrefix(c.Text(), "MethodGroup:") {
 					parts := strings.Split(c.Text(), "\n")
 					groupName := strings.TrimSpace(parts[0][12:])
@@ -367,15 +387,19 @@ func parseApiASTInfo(apiFile, iface string) (map[string]string, map[string]strin
 				}
 			}
 
-			last := cs[len(cs)-1].Text()
+			l := len(filteredComments) - 1
+			if len(filteredComments) > 1 {
+				l = len(filteredComments) - 2
+			}
+			last := filteredComments[l].Text()
 			if !strings.HasPrefix(last, "MethodGroup:") {
-				out[mn] = last
+				comments[mn] = last
 			} else {
-				out[mn] = noComment
+				comments[mn] = NoComment
 			}
 		}
 	}
-	return out, groupDocs
+	return comments, groupDocs
 }
 
 type MethodGroup struct {
@@ -391,7 +415,7 @@ type Method struct {
 	ResponseExample string
 }
 
-func methodGroupFromName(mn string) string {
+func MethodGroupFromName(mn string) string {
 	i := strings.IndexFunc(mn[1:], func(r rune) bool {
 		return unicode.IsUpper(r)
 	})
@@ -400,126 +424,3 @@ func methodGroupFromName(mn string) string {
 	}
 	return mn[:i+1]
 }
-
-func main() {
-	comments, groupComments := parseApiASTInfo(os.Args[1], os.Args[2])
-
-	groups := make(map[string]*MethodGroup)
-
-	var t reflect.Type
-	var permStruct, commonPermStruct reflect.Type
-
-	switch os.Args[2] {
-	case "FullNode":
-		t = reflect.TypeOf(new(struct{ api.FullNode })).Elem()
-		permStruct = reflect.TypeOf(apistruct.FullNodeStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
-	case "StorageMiner":
-		t = reflect.TypeOf(new(struct{ api.StorageMiner })).Elem()
-		permStruct = reflect.TypeOf(apistruct.StorageMinerStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.CommonStruct{}.Internal)
-	case "WorkerAPI":
-		t = reflect.TypeOf(new(struct{ api.WorkerAPI })).Elem()
-		permStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
-		commonPermStruct = reflect.TypeOf(apistruct.WorkerStruct{}.Internal)
-	default:
-		panic("unknown type")
-	}
-
-	for i := 0; i < t.NumMethod(); i++ {
-		m := t.Method(i)
-
-		groupName := methodGroupFromName(m.Name)
-
-		g, ok := groups[groupName]
-		if !ok {
-			g = new(MethodGroup)
-			g.Header = groupComments[groupName]
-			g.GroupName = groupName
-			groups[groupName] = g
-		}
-
-		var args []interface{}
-		ft := m.Func.Type()
-		for j := 2; j < ft.NumIn(); j++ {
-			inp := ft.In(j)
-			args = append(args, exampleValue(m.Name, inp, nil))
-		}
-
-		v, err := json.MarshalIndent(args, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		outv := exampleValue(m.Name, ft.Out(0), nil)
-
-		ov, err := json.MarshalIndent(outv, "", "  ")
-		if err != nil {
-			panic(err)
-		}
-
-		g.Methods = append(g.Methods, &Method{
-			Name:            m.Name,
-			Comment:         comments[m.Name],
-			InputExample:    string(v),
-			ResponseExample: string(ov),
-		})
-	}
-
-	var groupslice []*MethodGroup
-	for _, g := range groups {
-		groupslice = append(groupslice, g)
-	}
-
-	sort.Slice(groupslice, func(i, j int) bool {
-		return groupslice[i].GroupName < groupslice[j].GroupName
-	})
-
-	fmt.Printf("# Groups\n")
-
-	for _, g := range groupslice {
-		fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
-		for _, method := range g.Methods {
-			fmt.Printf("  * [%s](#%s)\n", method.Name, method.Name)
-		}
-	}
-
-	for _, g := range groupslice {
-		g := g
-		fmt.Printf("## %s\n", g.GroupName)
-		fmt.Printf("%s\n\n", g.Header)
-
-		sort.Slice(g.Methods, func(i, j int) bool {
-			return g.Methods[i].Name < g.Methods[j].Name
-		})
-
-		for _, m := range g.Methods {
-			fmt.Printf("### %s\n", m.Name)
-			fmt.Printf("%s\n\n", m.Comment)
-
-			meth, ok := permStruct.FieldByName(m.Name)
-			if !ok {
-				meth, ok = commonPermStruct.FieldByName(m.Name)
-				if !ok {
-					panic("no perms for method: " + m.Name)
-				}
-			}
-
-			perms := meth.Tag.Get("perm")
-
-			fmt.Printf("Perms: %s\n\n", perms)
-
-			if strings.Count(m.InputExample, "\n") > 0 {
-				fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
-			} else {
-				fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
-			}
-
-			if strings.Count(m.ResponseExample, "\n") > 0 {
-				fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
-			} else {
-				fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
-			}
-		}
-	}
-}

api/mocks/mock_full.go

@@ -18,6 +18,7 @@ import (
 	dline "github.com/filecoin-project/go-state-types/dline"
 	network "github.com/filecoin-project/go-state-types/network"
 	api "github.com/filecoin-project/lotus/api"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	miner "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	types "github.com/filecoin-project/lotus/chain/types"
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
@@ -783,6 +784,21 @@ func (mr *MockFullNodeMockRecorder) CreateBackup(arg0, arg1 interface{}) *gomock
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBackup", reflect.TypeOf((*MockFullNode)(nil).CreateBackup), arg0, arg1)
 }
 
+// Discover mocks base method
+func (m *MockFullNode) Discover(arg0 context.Context) (apitypes.OpenRPCDocument, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Discover", arg0)
+	ret0, _ := ret[0].(apitypes.OpenRPCDocument)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Discover indicates an expected call of Discover
+func (mr *MockFullNodeMockRecorder) Discover(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Discover", reflect.TypeOf((*MockFullNode)(nil).Discover), arg0)
+}
+
 // GasEstimateFeeCap mocks base method
 func (m *MockFullNode) GasEstimateFeeCap(arg0 context.Context, arg1 *types.Message, arg2 int64, arg3 types.TipSetKey) (big.Int, error) {
 	m.ctrl.T.Helper()

api/test/deals.go

@@ -23,9 +23,11 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/types"
 	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/impl"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
 	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
@@ -183,6 +185,71 @@ func TestPublishDealsBatching(t *testing.T, b APIBuilder, blocktime time.Duratio
 	}
 }
 
+func TestBatchDealInput(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
+	publishPeriod := 10 * time.Second
+	maxDealsPerMsg := uint64(4)
+
+	// Set max deals per publish deals message to maxDealsPerMsg
+	minerDef := []StorageMiner{{
+		Full: 0,
+		Opts: node.Options(
+			node.Override(
+				new(*storageadapter.DealPublisher),
+				storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{
+					Period:         publishPeriod,
+					MaxDealsPerMsg: maxDealsPerMsg,
+				})),
+			node.Override(new(dtypes.GetSealingConfigFunc), func() (dtypes.GetSealingConfigFunc, error) {
+				return func() (sealiface.Config, error) {
+					return sealiface.Config{
+						MaxWaitDealsSectors:       1,
+						MaxSealingSectors:         1,
+						MaxSealingSectorsForDeals: 2,
+						AlwaysKeepUnsealedCopy:    true,
+					}, nil
+				}, nil
+			}),
+		),
+		Preseal: PresealGenesis,
+	}}
+
+	// Create a connect client and miner node
+	n, sn := b(t, OneFull, minerDef)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+	s := connectAndStartMining(t, b, blocktime, client, miner)
+	defer s.blockMiner.Stop()
+
+	// Starts a deal and waits until it's published
+	runDealTillSeal := func(rseed int) {
+		res, _, err := CreateClientFile(s.ctx, s.client, rseed)
+		require.NoError(t, err)
+
+		dc := startDeal(t, s.ctx, s.miner, s.client, res.Root, false, startEpoch)
+		waitDealSealed(t, s.ctx, s.miner, s.client, dc, false)
+	}
+
+	// Run maxDealsPerMsg+1 deals in parallel
+	done := make(chan struct{}, maxDealsPerMsg+1)
+	for rseed := 1; rseed <= int(maxDealsPerMsg+1); rseed++ {
+		rseed := rseed
+		go func() {
+			runDealTillSeal(rseed)
+			done <- struct{}{}
+		}()
+	}
+
+	// Wait for maxDealsPerMsg of the deals to be published
+	for i := 0; i < int(maxDealsPerMsg); i++ {
+		<-done
+	}
+
+	sl, err := sn[0].SectorsList(s.ctx)
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, len(sl), 4)
+	require.LessOrEqual(t, len(sl), 5)
+}
+
 func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, startEpoch abi.ChainEpoch) {
 	s := setupOneClientOneMiner(t, b, blocktime)
 	defer s.blockMiner.Stop()

api/types/actors.go

@@ -0,0 +1,5 @@
+package apitypes
+
+import "github.com/filecoin-project/go-state-types/network"
+
+type NetworkVersion = network.Version

api/types/openrpc.go

@@ -0,0 +1,3 @@
+package apitypes
+
+type OpenRPCDocument map[string]interface{}

blockstore/badger/blockstore.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"runtime"
 	"sync/atomic"
 
 	"github.com/dgraph-io/badger/v2"
@@ -150,6 +151,20 @@ func (b *Blockstore) CollectGarbage() error {
 	return err
 }
 
+// Compact runs a synchronous compaction
+func (b *Blockstore) Compact() error {
+	if atomic.LoadInt64(&b.state) != stateOpen {
+		return ErrBlockstoreClosed
+	}
+
+	nworkers := runtime.NumCPU() / 2
+	if nworkers < 2 {
+		nworkers = 2
+	}
+
+	return b.DB.Flatten(nworkers)
+}
+
 // View implements blockstore.Viewer, which leverages zero-copy read-only
 // access to values.
 func (b *Blockstore) View(cid cid.Cid, fn func([]byte) error) error {
@@ -288,9 +303,6 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
 		return ErrBlockstoreClosed
 	}
 
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
 	// toReturn tracks the byte slices to return to the pool, if we're using key
 	// prefixing. we can't return each slice to the pool after each Set, because
 	// badger holds on to the slice.
@@ -304,6 +316,9 @@ func (b *Blockstore) PutMany(blocks []blocks.Block) error {
 		}()
 	}
 
+	batch := b.DB.NewWriteBatch()
+	defer batch.Cancel()
+
 	for _, block := range blocks {
 		k, pooled := b.PooledStorageKey(block.Cid())
 		if pooled {
@@ -342,9 +357,6 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
 		return ErrBlockstoreClosed
 	}
 
-	batch := b.DB.NewWriteBatch()
-	defer batch.Cancel()
-
 	// toReturn tracks the byte slices to return to the pool, if we're using key
 	// prefixing. we can't return each slice to the pool after each Set, because
 	// badger holds on to the slice.
@@ -358,6 +370,9 @@ func (b *Blockstore) DeleteMany(cids []cid.Cid) error {
 		}()
 	}
 
+	batch := b.DB.NewWriteBatch()
+	defer batch.Cancel()
+
 	for _, cid := range cids {
 		k, pooled := b.PooledStorageKey(cid)
 		if pooled {

blockstore/splitstore/splitstore.go

@@ -798,15 +798,26 @@ func (s *SplitStore) purgeTracking(cids []cid.Cid) error {
 }
 
 func (s *SplitStore) gcHotstore() {
+	if compact, ok := s.hot.(interface{ Compact() error }); ok {
+		log.Infof("compacting hotstore")
+		startCompact := time.Now()
+		err := compact.Compact()
+		if err != nil {
+			log.Warnf("error compacting hotstore: %s", err)
+			return
+		}
+		log.Infow("hotstore compaction done", "took", time.Since(startCompact))
+	}
+
 	if gc, ok := s.hot.(interface{ CollectGarbage() error }); ok {
 		log.Infof("garbage collecting hotstore")
 		startGC := time.Now()
 		err := gc.CollectGarbage()
 		if err != nil {
 			log.Warnf("error garbage collecting hotstore: %s", err)
-		} else {
+			return
-			log.Infow("garbage collection done", "took", time.Since(startGC))
 		}
+		log.Infow("hotstore garbage collection done", "took", time.Since(startGC))
 	}
 }
 

blockstore/timed.go

@@ -103,13 +103,20 @@ func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error {
 }
 
 func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error {
+	// The underlying blockstore is always a "mem" blockstore so there's no difference,
+	// from a performance perspective, between view & get. So we call Get to avoid
+	// calling an arbitrary callback while holding a lock.
 	t.mu.RLock()
-	defer t.mu.RUnlock()
+	block, err := t.active.Get(k)
-	err := t.active.View(k, callback)
 	if err == ErrNotFound {
-		err = t.inactive.View(k, callback)
+		block, err = t.inactive.Get(k)
 	}
-	return err
+	t.mu.RUnlock()
+
+	if err != nil {
+		return err
+	}
+	return callback(block.RawData())
 }
 
 func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error) {

build/bootstrap/mainnet.pi

@@ -7,7 +7,7 @@
 /dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ
 /dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf
 /dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR
-/dns4/lotus-bootstrap.forceup.cn/tcp/41778/p2p/12D3KooWFQsv3nRMUevZNWWsY1Wu6NUzUbawnWU5NcRhgKuJA37C
+/dns4/lotus-bootstrap.ipfsforce.com/tcp/41778/p2p/12D3KooWGhufNmZHF3sv48aQeS13ng5XVJZ9E6qy2Ms4VzqeUsHk
 /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz
 /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt

build/openrpc.go

@@ -0,0 +1,43 @@
+package build
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+
+	rice "github.com/GeertJohan/go.rice"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+)
+
+func mustReadGzippedOpenRPCDocument(data []byte) apitypes.OpenRPCDocument {
+	zr, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		log.Fatal(err)
+	}
+	m := apitypes.OpenRPCDocument{}
+	err = json.NewDecoder(zr).Decode(&m)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = zr.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	return m
+}
+
+func OpenRPCDiscoverJSON_Full() apitypes.OpenRPCDocument {
+	data := rice.MustFindBox("openrpc").MustBytes("full.json.gz")
+	return mustReadGzippedOpenRPCDocument(data)
+}
+
+func OpenRPCDiscoverJSON_Miner() apitypes.OpenRPCDocument {
+	data := rice.MustFindBox("openrpc").MustBytes("miner.json.gz")
+	return mustReadGzippedOpenRPCDocument(data)
+}
+
+func OpenRPCDiscoverJSON_Worker() apitypes.OpenRPCDocument {
+	data := rice.MustFindBox("openrpc").MustBytes("worker.json.gz")
+	return mustReadGzippedOpenRPCDocument(data)
+}

build/openrpc_test.go

@@ -0,0 +1,23 @@
+package build
+
+import (
+	"testing"
+
+	apitypes "github.com/filecoin-project/lotus/api/types"
+)
+
+func TestOpenRPCDiscoverJSON_Version(t *testing.T) {
+	// openRPCDocVersion is the current OpenRPC version of the API docs.
+	openRPCDocVersion := "1.2.6"
+
+	for i, docFn := range []func() apitypes.OpenRPCDocument{
+		OpenRPCDiscoverJSON_Full,
+		OpenRPCDiscoverJSON_Miner,
+		OpenRPCDiscoverJSON_Worker,
+	} {
+		doc := docFn()
+		if got, ok := doc["openrpc"]; !ok || got != openRPCDocVersion {
+			t.Fatalf("case: %d, want: %s, got: %v, doc: %v", i, openRPCDocVersion, got, doc)
+		}
+	}
+}

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.5.3"
+const BuildVersion = "1.6.0-dev"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit

chain/events/events.go

@@ -55,11 +55,11 @@ type Events struct {
 
 	heightEvents
 	*hcEvents
+
+	observers []TipSetObserver
 }
 
-func NewEvents(ctx context.Context, api eventAPI) *Events {
+func NewEventsWithConfidence(ctx context.Context, api eventAPI, gcConfidence abi.ChainEpoch) *Events {
-	gcConfidence := 2 * build.ForkLengthThreshold
-
 	tsc := newTSCache(gcConfidence, api)
 
 	e := &Events{
@@ -77,8 +77,9 @@ func NewEvents(ctx context.Context, api eventAPI) *Events {
 			htHeights:        map[abi.ChainEpoch][]uint64{},
 		},
 
-		hcEvents: newHCEvents(ctx, api, tsc, uint64(gcConfidence)),
+		hcEvents:  newHCEvents(ctx, api, tsc, uint64(gcConfidence)),
-		ready:    make(chan struct{}),
+		ready:     make(chan struct{}),
+		observers: []TipSetObserver{},
 	}
 
 	go e.listenHeadChanges(ctx)
@@ -92,6 +93,11 @@ func NewEvents(ctx context.Context, api eventAPI) *Events {
 	return e
 }
 
+func NewEvents(ctx context.Context, api eventAPI) *Events {
+	gcConfidence := 2 * build.ForkLengthThreshold
+	return NewEventsWithConfidence(ctx, api, gcConfidence)
+}
+
 func (e *Events) listenHeadChanges(ctx context.Context) {
 	for {
 		if err := e.listenHeadChangesOnce(ctx); err != nil {
@@ -164,7 +170,7 @@ func (e *Events) listenHeadChangesOnce(ctx context.Context) error {
 			}
 		}
 
-		if err := e.headChange(rev, app); err != nil {
+		if err := e.headChange(ctx, rev, app); err != nil {
 			log.Warnf("headChange failed: %s", err)
 		}
 
@@ -177,7 +183,7 @@ func (e *Events) listenHeadChangesOnce(ctx context.Context) error {
 	return nil
 }
 
-func (e *Events) headChange(rev, app []*types.TipSet) error {
+func (e *Events) headChange(ctx context.Context, rev, app []*types.TipSet) error {
 	if len(app) == 0 {
 		return xerrors.New("events.headChange expected at least one applied tipset")
 	}
@@ -189,5 +195,39 @@ func (e *Events) headChange(rev, app []*types.TipSet) error {
 		return err
 	}
 
+	if err := e.observeChanges(ctx, rev, app); err != nil {
+		return err
+	}
 	return e.processHeadChangeEvent(rev, app)
 }
+
+// A TipSetObserver receives notifications of tipsets
+type TipSetObserver interface {
+	Apply(ctx context.Context, ts *types.TipSet) error
+	Revert(ctx context.Context, ts *types.TipSet) error
+}
+
+// TODO: add a confidence level so we can have observers with difference levels of confidence
+func (e *Events) Observe(obs TipSetObserver) error {
+	e.lk.Lock()
+	defer e.lk.Unlock()
+	e.observers = append(e.observers, obs)
+	return nil
+}
+
+// observeChanges expects caller to hold e.lk
+func (e *Events) observeChanges(ctx context.Context, rev, app []*types.TipSet) error {
+	for _, ts := range rev {
+		for _, o := range e.observers {
+			_ = o.Revert(ctx, ts)
+		}
+	}
+
+	for _, ts := range app {
+		for _, o := range e.observers {
+			_ = o.Apply(ctx, ts)
+		}
+	}
+
+	return nil
+}

chain/gen/mining.go

@@ -15,7 +15,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.WalletAPI, bt *api.BlockTemplate) (*types.FullBlock, error) {
+func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w api.Wallet, bt *api.BlockTemplate) (*types.FullBlock, error) {
 
 	pts, err := sm.ChainStore().LoadTipSet(bt.Parents)
 	if err != nil {

chain/messagepool/selection_test.go

@@ -1333,7 +1333,7 @@ readLoop:
 	}
 
 	actorMap := make(map[address.Address]address.Address)
-	actorWallets := make(map[address.Address]api.WalletAPI)
+	actorWallets := make(map[address.Address]api.Wallet)
 
 	for _, m := range msgs {
 		baseNonce := baseNonces[m.Message.From]

chain/messagesigner/messagesigner.go

@@ -29,13 +29,13 @@ type MpoolNonceAPI interface {
 // MessageSigner keeps track of nonces per address, and increments the nonce
 // when signing a message
 type MessageSigner struct {
-	wallet api.WalletAPI
+	wallet api.Wallet
 	lk     sync.Mutex
 	mpool  MpoolNonceAPI
 	ds     datastore.Batching
 }
 
-func NewMessageSigner(wallet api.WalletAPI, mpool MpoolNonceAPI, ds dtypes.MetadataDS) *MessageSigner {
+func NewMessageSigner(wallet api.Wallet, mpool MpoolNonceAPI, ds dtypes.MetadataDS) *MessageSigner {
 	ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/"))
 	return &MessageSigner{
 		wallet: wallet,

chain/stmgr/rpc/rpcstatemanager.go

@@ -1,4 +1,4 @@
-package modules
+package rpcstmgr
 
 import (
 	"context"
@@ -16,11 +16,11 @@ import (
 )
 
 type RPCStateManager struct {
-	gapi   api.GatewayAPI
+	gapi   api.Gateway
 	cstore *cbor.BasicIpldStore
 }
 
-func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager {
+func NewRPCStateManager(api api.Gateway) *RPCStateManager {
 	cstore := cbor.NewCborStore(blockstore.NewAPIBlockstore(api))
 	return &RPCStateManager{gapi: api, cstore: cstore}
 }

chain/wallet/ledger/ledger.go

@@ -36,7 +36,7 @@ type LedgerKeyInfo struct {
 	Path    []uint32
 }
 
-var _ api.WalletAPI = (*LedgerWallet)(nil)
+var _ api.Wallet = (*LedgerWallet)(nil)
 
 func (lw LedgerWallet) WalletSign(ctx context.Context, signer address.Address, toSign []byte, meta api.MsgMeta) (*crypto.Signature, error) {
 	ki, err := lw.getKeyInfo(signer)
@@ -227,7 +227,7 @@ func (lw LedgerWallet) WalletNew(ctx context.Context, t types.KeyType) (address.
 	return lw.importKey(lki)
 }
 
-func (lw *LedgerWallet) Get() api.WalletAPI {
+func (lw *LedgerWallet) Get() api.Wallet {
 	if lw == nil {
 		return nil
 	}

chain/wallet/multi.go

@@ -24,13 +24,13 @@ type MultiWallet struct {
 }
 
 type getif interface {
-	api.WalletAPI
+	api.Wallet
 
 	// workaround for the fact that iface(*struct(nil)) != nil
-	Get() api.WalletAPI
+	Get() api.Wallet
 }
 
-func firstNonNil(wallets ...getif) api.WalletAPI {
+func firstNonNil(wallets ...getif) api.Wallet {
 	for _, w := range wallets {
 		if w.Get() != nil {
 			return w
@@ -40,8 +40,8 @@ func firstNonNil(wallets ...getif) api.WalletAPI {
 	return nil
 }
 
-func nonNil(wallets ...getif) []api.WalletAPI {
+func nonNil(wallets ...getif) []api.Wallet {
-	var out []api.WalletAPI
+	var out []api.Wallet
 	for _, w := range wallets {
 		if w.Get() == nil {
 			continue
@@ -53,7 +53,7 @@ func nonNil(wallets ...getif) []api.WalletAPI {
 	return out
 }
 
-func (m MultiWallet) find(ctx context.Context, address address.Address, wallets ...getif) (api.WalletAPI, error) {
+func (m MultiWallet) find(ctx context.Context, address address.Address, wallets ...getif) (api.Wallet, error) {
 	ws := nonNil(wallets...)
 
 	for _, w := range ws {
@@ -167,4 +167,4 @@ func (m MultiWallet) WalletDelete(ctx context.Context, address address.Address)
 	}
 }
 
-var _ api.WalletAPI = MultiWallet{}
+var _ api.Wallet = MultiWallet{}

chain/wallet/remotewallet/remote.go

@@ -13,7 +13,7 @@ import (
 )
 
 type RemoteWallet struct {
-	api.WalletAPI
+	api.Wallet
 }
 
 func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle) (*RemoteWallet, error) {
@@ -41,7 +41,7 @@ func SetupRemoteWallet(info string) func(mctx helpers.MetricsCtx, lc fx.Lifecycl
 	}
 }
 
-func (w *RemoteWallet) Get() api.WalletAPI {
+func (w *RemoteWallet) Get() api.Wallet {
 	if w == nil {
 		return nil
 	}

chain/wallet/wallet.go

@@ -329,7 +329,7 @@ func (w *LocalWallet) WalletDelete(ctx context.Context, addr address.Address) er
 	return nil
 }
 
-func (w *LocalWallet) Get() api.WalletAPI {
+func (w *LocalWallet) Get() api.Wallet {
 	if w == nil {
 		return nil
 	}
@@ -337,7 +337,7 @@ func (w *LocalWallet) Get() api.WalletAPI {
 	return w
 }
 
-var _ api.WalletAPI = &LocalWallet{}
+var _ api.Wallet = &LocalWallet{}
 
 func swapMainnetForTestnetPrefix(addr string) (string, error) {
 	aChars := []rune(addr)

cli/util/api.go

@@ -230,7 +230,7 @@ func GetStorageMinerAPI(ctx *cli.Context, opts ...GetStorageMinerOption) (api.St
 	return client.NewStorageMinerRPC(ctx.Context, addr, headers)
 }
 
-func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error) {
+func GetWorkerAPI(ctx *cli.Context) (api.Worker, jsonrpc.ClientCloser, error) {
 	addr, headers, err := GetRawAPI(ctx, repo.Worker)
 	if err != nil {
 		return nil, nil, err
@@ -239,7 +239,7 @@ func GetWorkerAPI(ctx *cli.Context) (api.WorkerAPI, jsonrpc.ClientCloser, error)
 	return client.NewWorkerRPC(ctx.Context, addr, headers)
 }
 
-func GetGatewayAPI(ctx *cli.Context) (api.GatewayAPI, jsonrpc.ClientCloser, error) {
+func GetGatewayAPI(ctx *cli.Context) (api.Gateway, jsonrpc.ClientCloser, error) {
 	addr, headers, err := GetRawAPI(ctx, repo.FullNode)
 	if err != nil {
 		return nil, nil, err

cmd/lotus-gateway/api.go

@@ -404,7 +404,7 @@ func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []
 	return sigs.Verify(sig, k, msg) == nil, nil
 }
 
-var _ api.GatewayAPI = (*GatewayAPI)(nil)
+var _ api.Gateway = (*GatewayAPI)(nil)
 var _ full.ChainModuleAPI = (*GatewayAPI)(nil)
 var _ full.GasModuleAPI = (*GatewayAPI)(nil)
 var _ full.MpoolModuleAPI = (*GatewayAPI)(nil)

cmd/lotus-gateway/endtoend_test.go

@@ -249,12 +249,12 @@ func startNodes(
 				require.NoError(t, err)
 
 				// Create a gateway client API that connects to the gateway server
-				var gapi api.GatewayAPI
+				var gapi api.Gateway
 				gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil)
 				require.NoError(t, err)
 
 				// Provide the gateway API to dependency injection
-				return node.Override(new(api.GatewayAPI), gapi)
+				return node.Override(new(api.Gateway), gapi)
 			},
 		},
 	)

cmd/lotus-seal-worker/rpc.go

@@ -9,6 +9,8 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
+	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/build"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
@@ -76,4 +78,8 @@ func (w *worker) Session(ctx context.Context) (uuid.UUID, error) {
 	return w.LocalWorker.Session(ctx)
 }
 
+func (w *worker) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
+	return build.OpenRPCDiscoverJSON_Worker(), nil
+}
+
 var _ storiface.WorkerCalls = &worker{}

cmd/lotus-seal-worker/tasks.go

@@ -41,17 +41,17 @@ var tasksEnableCmd = &cli.Command{
 	Name:      "enable",
 	Usage:     "Enable a task type",
 	ArgsUsage: "[" + settableStr + "]",
-	Action:    taskAction(api.WorkerAPI.TaskEnable),
+	Action:    taskAction(api.Worker.TaskEnable),
 }
 
 var tasksDisableCmd = &cli.Command{
 	Name:      "disable",
 	Usage:     "Disable a task type",
 	ArgsUsage: "[" + settableStr + "]",
-	Action:    taskAction(api.WorkerAPI.TaskDisable),
+	Action:    taskAction(api.Worker.TaskDisable),
 }
 
-func taskAction(tf func(a api.WorkerAPI, ctx context.Context, tt sealtasks.TaskType) error) func(cctx *cli.Context) error {
+func taskAction(tf func(a api.Worker, ctx context.Context, tt sealtasks.TaskType) error) func(cctx *cli.Context) error {
 	return func(cctx *cli.Context) error {
 		if cctx.NArg() != 1 {
 			return xerrors.Errorf("expected 1 argument")

cmd/lotus-shed/main.go

@@ -37,6 +37,7 @@ func main() {
 		mpoolCmd,
 		genesisVerifyCmd,
 		mathCmd,
+		minerCmd,
 		mpoolStatsCmd,
 		exportChainCmd,
 		consensusCmd,

cmd/lotus-shed/miner.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerCmd = &cli.Command{
+	Name:  "miner",
+	Usage: "miner-related utilities",
+	Subcommands: []*cli.Command{
+		minerUnpackInfoCmd,
+	},
+}
+
+var minerUnpackInfoCmd = &cli.Command{
+	Name:      "unpack-info",
+	Usage:     "unpack miner info all dump",
+	ArgsUsage: "[allinfo.txt] [dir]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 2 {
+			return xerrors.Errorf("expected 2 args")
+		}
+
+		src, err := homedir.Expand(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("expand src: %w", err)
+		}
+
+		f, err := os.Open(src)
+		if err != nil {
+			return xerrors.Errorf("open file: %w", err)
+		}
+		defer f.Close() // nolint
+
+		dest, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("expand dest: %w", err)
+		}
+
+		var outf *os.File
+
+		r := bufio.NewReader(f)
+		for {
+			l, _, err := r.ReadLine()
+			if err == io.EOF {
+				if outf != nil {
+					return outf.Close()
+				}
+			}
+			if err != nil {
+				return xerrors.Errorf("read line: %w", err)
+			}
+			sl := string(l)
+
+			if strings.HasPrefix(sl, "#") {
+				if strings.Contains(sl, "..") {
+					return xerrors.Errorf("bad name %s", sl)
+				}
+
+				if strings.HasPrefix(sl, "#: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, sl[len("#: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+
+				if strings.HasPrefix(sl, "##: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, "Per Sector Infos", sl[len("##: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+			}
+
+			if outf != nil {
+				if _, err := outf.Write(l); err != nil {
+					return xerrors.Errorf("write line: %w", err)
+				}
+				if _, err := outf.Write([]byte("\n")); err != nil {
+					return xerrors.Errorf("write line end: %w", err)
+				}
+			}
+		}
+	},
+}

cmd/lotus-storage-miner/info_all.go

@@ -35,80 +35,80 @@ var infoAllCmd = &cli.Command{
 
 		fmt.Println("#: Version")
 		if err := lcli.VersionCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Miner Info")
 		if err := infoCmdAct(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Verbose info
 
 		fmt.Println("\n#: Storage List")
 		if err := storageListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Worker List")
 		if err := sealingWorkersCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: PeerID")
 		if err := lcli.NetId.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Listen Addresses")
 		if err := lcli.NetListen.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Reachability")
 		if err := lcli.NetReachability.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Very Verbose info
 		fmt.Println("\n#: Peers")
 		if err := lcli.NetPeers.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sealing Jobs")
 		if err := sealingJobsCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sched Diag")
 		if err := sealingSchedDiagCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Storage Ask")
 		if err := getAskCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Storage Deals")
 		if err := dealsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Retrieval Deals")
 		if err := retrievalDealsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sector List")
 		if err := sectorsListCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		fmt.Println("\n#: Sector Refs")
 		if err := sectorsRefsCmd.Action(cctx); err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		// Very Very Verbose info
@@ -116,7 +116,7 @@ var infoAllCmd = &cli.Command{
 
 		list, err := nodeApi.SectorsList(ctx)
 		if err != nil {
-			return err
+			fmt.Println("ERROR: ", err)
 		}
 
 		sort.Slice(list, func(i, j int) bool {
@@ -129,11 +129,11 @@ var infoAllCmd = &cli.Command{
 			fs := &flag.FlagSet{}
 			for _, f := range sectorsStatusCmd.Flags {
 				if err := f.Apply(fs); err != nil {
-					return err
+					fmt.Println("ERROR: ", err)
 				}
 			}
 			if err := fs.Parse([]string{"--log", "--on-chain-info", fmt.Sprint(s)}); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 
 			if err := sectorsStatusCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
@@ -144,7 +144,7 @@ var infoAllCmd = &cli.Command{
 
 			fs = &flag.FlagSet{}
 			if err := fs.Parse([]string{fmt.Sprint(s)}); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 
 			if err := storageFindCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
@@ -155,7 +155,7 @@ var infoAllCmd = &cli.Command{
 		if !_test {
 			fmt.Println("\n#: Goroutines")
 			if err := lcli.PprofGoroutines.Action(cctx); err != nil {
-				return err
+				fmt.Println("ERROR: ", err)
 			}
 		}
 

cmd/lotus-wallet/interactive.go

@@ -31,7 +31,7 @@ type InteractiveWallet struct {
 	lk sync.Mutex
 
 	apiGetter func() (api.FullNode, jsonrpc.ClientCloser, error)
-	under     api.WalletAPI
+	under     api.Wallet
 }
 
 func (c *InteractiveWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) {

cmd/lotus-wallet/logged.go

@@ -16,7 +16,7 @@ import (
 )
 
 type LoggedWallet struct {
-	under api.WalletAPI
+	under api.Wallet
 }
 
 func (c *LoggedWallet) WalletNew(ctx context.Context, typ types.KeyType) (address.Address, error) {

cmd/lotus-wallet/main.go

@@ -130,7 +130,7 @@ var runCmd = &cli.Command{
 			return err
 		}
 
-		var w api.WalletAPI = lw
+		var w api.Wallet = lw
 		if cctx.Bool("ledger") {
 			ds, err := lr.Datastore(context.Background(), "/metadata")
 			if err != nil {

cmd/lotus/daemon.go

@@ -289,7 +289,7 @@ var DaemonCmd = &cli.Command{
 
 		shutdownChan := make(chan struct{})
 
-		// If the daemon is started in "lite mode", provide a  GatewayAPI
+		// If the daemon is started in "lite mode", provide a  Gateway
 		// for RPC calls
 		liteModeDeps := node.Options()
 		if isLite {
@@ -299,7 +299,7 @@ var DaemonCmd = &cli.Command{
 			}
 
 			defer closer()
-			liteModeDeps = node.Override(new(api.GatewayAPI), gapi)
+			liteModeDeps = node.Override(new(api.Gateway), gapi)
 		}
 
 		// some libraries like ipfs/go-ds-measure and ipfs/go-ipfs-blockstore

cmd/lotus/rpc.go

@@ -37,6 +37,7 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut
 	}
 	rpcServer := jsonrpc.NewServer(serverOptions...)
 	rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(metrics.MetricedFullAPI(a)))
+	rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
 
 	ah := &auth.Handler{
 		Verify: a.AuthVerify,

documentation/en/api-methods-miner.md

@@ -1,6 +1,7 @@
 # Groups
 * [](#)
   * [Closing](#Closing)
+  * [Discover](#Discover)
   * [Session](#Session)
   * [Shutdown](#Shutdown)
   * [Version](#Version)
@@ -142,6 +143,25 @@ Inputs: `null`
 
 Response: `{}`
 
+### Discover
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "info": {
+    "title": "Lotus RPC API",
+    "version": "1.2.1/generated=2020-11-22T08:22:42-06:00"
+  },
+  "methods": [],
+  "openrpc": "1.2.6"
+}
+```
+
 ### Session
 
 
@@ -180,7 +200,7 @@ Response:
 
 
 ### ActorAddress
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -189,7 +209,7 @@ Inputs: `null`
 Response: `"f01234"`
 
 ### ActorAddressConfig
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -207,7 +227,7 @@ Response:
 ```
 
 ### ActorSectorSize
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -255,7 +275,7 @@ Response: `null`
 
 
 ### CheckProvable
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -300,63 +320,63 @@ Response: `{}`
 
 
 ### DealsConsiderOfflineRetrievalDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsConsiderOfflineStorageDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsConsiderOnlineRetrievalDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsConsiderOnlineStorageDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsConsiderUnverifiedStorageDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsConsiderVerifiedStorageDeals
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `true`
 
 ### DealsImportData
-There are not yet any comments for this method.
 
-Perms: write
+
+Perms: admin
 
 Inputs:
 ```json
@@ -371,25 +391,25 @@ Inputs:
 Response: `{}`
 
 ### DealsList
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `null`
 
 ### DealsPieceCidBlocklist
-There are not yet any comments for this method.
 
-Perms: read
+
+Perms: admin
 
 Inputs: `null`
 
 Response: `null`
 
 ### DealsSetConsiderOfflineRetrievalDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -403,7 +423,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetConsiderOfflineStorageDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -417,7 +437,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetConsiderOnlineRetrievalDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -431,7 +451,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetConsiderOnlineStorageDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -445,7 +465,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetConsiderUnverifiedStorageDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -459,7 +479,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetConsiderVerifiedStorageDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -473,7 +493,7 @@ Inputs:
 Response: `{}`
 
 ### DealsSetPieceCidBlocklist
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -546,7 +566,7 @@ Inputs:
 Response: `{}`
 
 ### MarketDataTransferUpdates
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -570,7 +590,7 @@ Response:
 ```
 
 ### MarketGetAsk
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -597,7 +617,7 @@ Response:
 ```
 
 ### MarketGetDealUpdates
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -662,7 +682,7 @@ Response:
 ```
 
 ### MarketGetRetrievalAsk
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -679,7 +699,7 @@ Response:
 ```
 
 ### MarketImportDealData
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -696,7 +716,7 @@ Inputs:
 Response: `{}`
 
 ### MarketListDataTransfers
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -705,7 +725,7 @@ Inputs: `null`
 Response: `null`
 
 ### MarketListDeals
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -714,7 +734,7 @@ Inputs: `null`
 Response: `null`
 
 ### MarketListIncompleteDeals
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -723,7 +743,7 @@ Inputs: `null`
 Response: `null`
 
 ### MarketListRetrievalDeals
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -732,7 +752,7 @@ Inputs: `null`
 Response: `null`
 
 ### MarketPendingDeals
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -748,7 +768,7 @@ Response:
 ```
 
 ### MarketPublishPendingDeals
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -774,7 +794,7 @@ Inputs:
 Response: `{}`
 
 ### MarketSetAsk
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -792,7 +812,7 @@ Inputs:
 Response: `{}`
 
 ### MarketSetRetrievalAsk
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -814,7 +834,7 @@ Response: `{}`
 
 
 ### MiningBase
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1102,7 +1122,7 @@ Response: `null`
 
 
 ### PiecesGetCIDInfo
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1126,7 +1146,7 @@ Response:
 ```
 
 ### PiecesGetPieceInfo
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1150,7 +1170,7 @@ Response:
 ```
 
 ### PiecesListCidInfos
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1159,7 +1179,7 @@ Inputs: `null`
 Response: `null`
 
 ### PiecesListPieces
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1190,6 +1210,7 @@ Response:
 
 
 ### ReturnAddPiece
+storiface.WorkerReturn
 
 
 Perms: admin
@@ -1475,7 +1496,7 @@ Response: `{}`
 
 
 ### SealingAbort
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -1534,7 +1555,7 @@ Inputs: `null`
 Response: `60000000000`
 
 ### SectorMarkForUpgrade
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -1688,7 +1709,7 @@ Response:
 ```
 
 ### SectorsRefs
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -1771,7 +1792,7 @@ Response:
 ```
 
 ### SectorsUpdate
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -1789,7 +1810,7 @@ Response: `{}`
 
 
 ### StorageAddLocal
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -1803,6 +1824,7 @@ Inputs:
 Response: `{}`
 
 ### StorageAttach
+stores.SectorIndex
 
 
 Perms: admin
@@ -1931,7 +1953,7 @@ Response:
 ```
 
 ### StorageList
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -1951,7 +1973,7 @@ Response:
 ```
 
 ### StorageLocal
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -2009,7 +2031,7 @@ Inputs:
 Response: `{}`
 
 ### StorageStat
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -2070,7 +2092,7 @@ Inputs:
 Response: `{}`
 
 ### WorkerJobs
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -2102,7 +2124,7 @@ Response:
 ```
 
 ### WorkerStats
-There are not yet any comments for this method.
+
 
 Perms: admin
 

documentation/en/api-methods-worker.md

@@ -40,7 +40,7 @@
 
 
 ### Enabled
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -81,7 +81,7 @@ Response:
 ```
 
 ### Info
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -102,7 +102,7 @@ Response:
 ```
 
 ### Paths
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -139,7 +139,6 @@ Inputs: `null`
 Response: `"07070707-0707-0707-0707-070707070707"`
 
 ### Version
-TODO: Info() (name, ...) ?
 
 
 Perms: admin
@@ -152,6 +151,7 @@ Response: `65792`
 
 
 ### AddPiece
+storiface.WorkerCalls
 
 
 Perms: admin
@@ -488,7 +488,7 @@ Response: `{}`
 
 
 ### StorageAddLocal
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -505,7 +505,7 @@ Response: `{}`
 
 
 ### TaskDisable
-There are not yet any comments for this method.
+
 
 Perms: admin
 
@@ -519,7 +519,7 @@ Inputs:
 Response: `{}`
 
 ### TaskEnable
-There are not yet any comments for this method.
+
 
 Perms: admin
 

documentation/en/api-methods.md

@@ -1,6 +1,7 @@
 # Groups
 * [](#)
   * [Closing](#Closing)
+  * [Discover](#Discover)
   * [Session](#Session)
   * [Shutdown](#Shutdown)
   * [Version](#Version)
@@ -226,6 +227,25 @@ Inputs: `null`
 
 Response: `{}`
 
+### Discover
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "info": {
+    "title": "Lotus RPC API",
+    "version": "1.2.1/generated=2020-11-22T08:22:42-06:00"
+  },
+  "methods": [],
+  "openrpc": "1.2.6"
+}
+```
+
 ### Session
 
 
@@ -425,6 +445,17 @@ Response:
 ### ChainGetBlockMessages
 ChainGetBlockMessages returns messages stored in the specified block.
 
+Note: If there are multiple blocks in a tipset, it's likely that some
+messages will be duplicated. It's also possible for blocks in a tipset to have
+different messages from the same sender at the same nonce. When that happens,
+only the first message (in a block with lowest ticket) will be considered
+for execution
+
+NOTE: THIS METHOD SHOULD ONLY BE USED FOR GETTING MESSAGES IN A SPECIFIC BLOCK
+
+DO NOT USE THIS METHOD TO GET MESSAGES INCLUDED IN A TIPSET
+Use ChainGetParentMessages, which will perform correct message deduplication
+
 
 Perms: read
 
@@ -499,7 +530,7 @@ Response:
 ```
 
 ### ChainGetNode
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -540,7 +571,8 @@ Response: `null`
 
 ### ChainGetParentReceipts
 ChainGetParentReceipts returns receipts for messages in parent tipset of
-the specified block.
+the specified block. The receipts in the list returned is one-to-one with the
+messages returned by a call to ChainGetParentMessages with the same blockCid.
 
 
 Perms: read
@@ -853,7 +885,7 @@ retrieval markets as a client
 ClientCalcCommP calculates the CommP for a specified file
 
 
-Perms: read
+Perms: write
 
 Inputs:
 ```json
@@ -890,7 +922,7 @@ Inputs:
 Response: `{}`
 
 ### ClientDataTransferUpdates
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -1081,7 +1113,7 @@ Response: `"string value"`
 ClientGetDealUpdates returns the status of updated deals
 
 
-Perms: read
+Perms: write
 
 Inputs: `null`
 
@@ -1762,7 +1794,7 @@ Response:
 
 
 ### MinerCreateBlock
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -2178,7 +2210,7 @@ Response: `null`
 MpoolSetConfig sets the mpool config to (a copy of) the supplied config
 
 
-Perms: write
+Perms: admin
 
 Inputs:
 ```json
@@ -2197,7 +2229,7 @@ Inputs:
 Response: `{}`
 
 ### MpoolSub
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -2943,7 +2975,7 @@ The Paych methods are for interacting with and managing payment channels
 
 
 ### PaychAllocateLane
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -2957,7 +2989,7 @@ Inputs:
 Response: `42`
 
 ### PaychAvailableFunds
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -2983,7 +3015,7 @@ Response:
 ```
 
 ### PaychAvailableFundsByFromTo
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3010,7 +3042,7 @@ Response:
 ```
 
 ### PaychCollect
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3053,7 +3085,7 @@ Response:
 ```
 
 ### PaychGetWaitReady
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3069,7 +3101,7 @@ Inputs:
 Response: `"f01234"`
 
 ### PaychList
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -3078,7 +3110,7 @@ Inputs: `null`
 Response: `null`
 
 ### PaychNewPayment
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3103,7 +3135,7 @@ Response:
 ```
 
 ### PaychSettle
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3122,7 +3154,7 @@ Response:
 ```
 
 ### PaychStatus
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -3142,7 +3174,7 @@ Response:
 ```
 
 ### PaychVoucherAdd
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -3178,7 +3210,7 @@ Inputs:
 Response: `"0"`
 
 ### PaychVoucherCheckSpendable
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -3214,7 +3246,7 @@ Inputs:
 Response: `true`
 
 ### PaychVoucherCheckValid
-There are not yet any comments for this method.
+
 
 Perms: read
 
@@ -3248,7 +3280,7 @@ Inputs:
 Response: `{}`
 
 ### PaychVoucherCreate
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3289,7 +3321,7 @@ Response:
 ```
 
 ### PaychVoucherList
-There are not yet any comments for this method.
+
 
 Perms: write
 
@@ -3303,7 +3335,7 @@ Inputs:
 Response: `null`
 
 ### PaychVoucherSubmit
-There are not yet any comments for this method.
+
 
 Perms: sign
 
@@ -3565,6 +3597,36 @@ Response: `"0"`
 StateCompute is a flexible command that applies the given messages on the given tipset.
 The messages are run as though the VM were at the provided height.
 
+When called, StateCompute will:
+- Load the provided tipset, or use the current chain head if not provided
+- Compute the tipset state of the provided tipset on top of the parent state
+  - (note that this step runs before vmheight is applied to the execution)
+  - Execute state upgrade if any were scheduled at the epoch, or in null
+    blocks preceding the tipset
+  - Call the cron actor on null blocks preceding the tipset
+  - For each block in the tipset
+    - Apply messages in blocks in the specified
+    - Award block reward by calling the reward actor
+  - Call the cron actor for the current epoch
+- If the specified vmheight is higher than the current epoch, apply any
+  needed state upgrades to the state
+- Apply the specified messages to the state
+
+The vmheight parameter sets VM execution epoch, and can be used to simulate
+message execution in different network versions. If the specified vmheight
+epoch is higher than the epoch of the specified tipset, any state upgrades
+until the vmheight will be executed on the state before applying messages
+specified by the user.
+
+Note that the initial tipset state computation is not affected by the
+vmheight parameter - only the messages in the `apply` set are
+
+If the caller wants to simply compute the state, vmheight should be set to
+the epoch of the specified tipset.
+
+Messages in the `apply` parameter must have the correct nonces, and gas
+values set.
+
 
 Perms: read
 
@@ -3686,7 +3748,15 @@ Response:
 ```
 
 ### StateGetReceipt
-StateGetReceipt returns the message receipt for the given message
+StateGetReceipt returns the message receipt for the given message or for a
+matching gas-repriced replacing message
+
+NOTE: If the requested message was replaced, this method will return the receipt
+for the replacing message - if the caller needs the receipt for exactly the
+requested message, use StateSearchMsg().Receipt, and check that MsgLookup.Message
+is matching the requested CID
+
+DEPRECATED: Use StateSearchMsg, this method won't be supported in v1 API
 
 
 Perms: read
@@ -4450,7 +4520,22 @@ Response:
 
 ### StateReplay
 StateReplay replays a given message, assuming it was included in a block in the specified tipset.
-If no tipset key is provided, the appropriate tipset is looked up.
+
+If a tipset key is provided, and a replacing message is found on chain,
+the method will return an error saying that the message wasn't found
+
+If no tipset key is provided, the appropriate tipset is looked up, and if
+the message was gas-repriced, the on-chain message will be replayed - in
+that case the returned InvocResult.MsgCid will not match the Cid param
+
+If the caller wants to ensure that exactly the requested message was executed,
+they MUST check that InvocResult.MsgCid is equal to the provided Cid.
+Without this check both the requested and original message may appear as
+successfully executed on-chain, which may look like a double-spend.
+
+A replacing message is a message with a different CID, any of Gas values, and
+different signature, but with all other parameters matching (source/destination,
+nonce, params, etc.)
 
 
 Perms: read
@@ -4544,6 +4629,20 @@ Response:
 ### StateSearchMsg
 StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
 
+NOTE: If a replacing message is found on chain, this method will return
+a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+result of the execution of the replacing message.
+
+If the caller wants to ensure that exactly the requested message was executed,
+they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+Without this check both the requested and original message may appear as
+successfully executed on-chain, which may look like a double-spend.
+
+A replacing message is a message with a different CID, any of Gas values, and
+different signature, but with all other parameters matching (source/destination,
+nonce, params, etc.)
+
 
 Perms: read
 
@@ -4583,6 +4682,20 @@ Response:
 ### StateSearchMsgLimited
 StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
 
+NOTE: If a replacing message is found on chain, this method will return
+a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+result of the execution of the replacing message.
+
+If the caller wants to ensure that exactly the requested message was executed,
+they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+Without this check both the requested and original message may appear as
+successfully executed on-chain, which may look like a double-spend.
+
+A replacing message is a message with a different CID, any of Gas values, and
+different signature, but with all other parameters matching (source/destination,
+nonce, params, etc.)
+
 
 Perms: read
 
@@ -4877,6 +4990,20 @@ Response: `"0"`
 StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
 message arrives on chain, and gets to the indicated confidence depth.
 
+NOTE: If a replacing message is found on chain, this method will return
+a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+result of the execution of the replacing message.
+
+If the caller wants to ensure that exactly the requested message was executed,
+they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+Without this check both the requested and original message may appear as
+successfully executed on-chain, which may look like a double-spend.
+
+A replacing message is a message with a different CID, any of Gas values, and
+different signature, but with all other parameters matching (source/destination,
+nonce, params, etc.)
+
 
 Perms: read
 
@@ -4919,6 +5046,20 @@ StateWaitMsgLimited looks back up to limit epochs in the chain for a message.
 If not found, it blocks until the message arrives on chain, and gets to the
 indicated confidence depth.
 
+NOTE: If a replacing message is found on chain, this method will return
+a MsgLookup for the replacing message - the MsgLookup.Message will be a different
+CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
+result of the execution of the replacing message.
+
+If the caller wants to ensure that exactly the requested message was executed,
+they MUST check that MsgLookup.Message is equal to the provided 'cid'.
+Without this check both the requested and original message may appear as
+successfully executed on-chain, which may look like a double-spend.
+
+A replacing message is a message with a different CID, any of Gas values, and
+different signature, but with all other parameters matching (source/destination,
+nonce, params, etc.)
+
 
 Perms: read
 
@@ -5219,7 +5360,7 @@ Response: `"f01234"`
 WalletDelete deletes an address from the wallet.
 
 
-Perms: write
+Perms: admin
 
 Inputs:
 ```json
@@ -5315,7 +5456,7 @@ Response: `"f01234"`
 WalletSetDefault marks the given address as as the default one.
 
 
-Perms: admin
+Perms: write
 
 Inputs:
 ```json

extern/sector-storage/mock/mock.go

@@ -325,6 +325,8 @@ func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorI
 }
 
 func generateFakePoStProof(sectorInfo []proof2.SectorInfo, randomness abi.PoStRandomness) []byte {
+	randomness[31] &= 0x3f
+
 	hasher := sha256.New()
 	_, _ = hasher.Write(randomness)
 	for _, info := range sectorInfo {
@@ -489,6 +491,7 @@ func (m mockVerif) VerifySeal(svi proof2.SealVerifyInfo) (bool, error) {
 }
 
 func (m mockVerif) VerifyWinningPoSt(ctx context.Context, info proof2.WinningPoStVerifyInfo) (bool, error) {
+	info.Randomness[31] &= 0x3f
 	return true, nil
 }
 

extern/storage-sealing/fsm.go

@@ -300,7 +300,9 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 
 	*/
 
-	m.stats.updateSector(m.minerSectorID(state.SectorNumber), state.State)
+	if err := m.onUpdateSector(context.TODO(), state); err != nil {
+		log.Errorw("update sector stats", "error", err)
+	}
 
 	switch state.State {
 	// Happy path
@@ -391,6 +393,37 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 	return nil, processed, nil
 }
 
+func (m *Sealing) onUpdateSector(ctx context.Context, state *SectorInfo) error {
+	if m.getConfig == nil {
+		return nil // tests
+	}
+
+	cfg, err := m.getConfig()
+	if err != nil {
+		return xerrors.Errorf("getting config: %w", err)
+	}
+	sp, err := m.currentSealProof(ctx)
+	if err != nil {
+		return xerrors.Errorf("getting seal proof type: %w", err)
+	}
+
+	shouldUpdateInput := m.stats.updateSector(cfg, m.minerSectorID(state.SectorNumber), state.State)
+
+	// trigger more input processing when we've dipped below max sealing limits
+	if shouldUpdateInput {
+		go func() {
+			m.inputLk.Lock()
+			defer m.inputLk.Unlock()
+
+			if err := m.updateInput(ctx, sp); err != nil {
+				log.Errorf("%+v", err)
+			}
+		}()
+	}
+
+	return nil
+}
+
 func planCommitting(events []statemachine.Event, state *SectorInfo) (uint64, error) {
 	for i, event := range events {
 		switch e := event.User.(type) {

extern/storage-sealing/garbage.go

@@ -28,18 +28,13 @@ func (m *Sealing) PledgeSector(ctx context.Context) (storage.SectorRef, error) {
 		return storage.SectorRef{}, xerrors.Errorf("getting seal proof type: %w", err)
 	}
 
-	sid, err := m.sc.Next()
+	sid, err := m.createSector(ctx, cfg, spt)
 	if err != nil {
-		return storage.SectorRef{}, xerrors.Errorf("generating sector number: %w", err)
+		return storage.SectorRef{}, err
-	}
-	sectorID := m.minerSector(spt, sid)
-	err = m.sealer.NewSector(ctx, sectorID)
-	if err != nil {
-		return storage.SectorRef{}, xerrors.Errorf("notifying sealer of the new sector: %w", err)
 	}
 
 	log.Infof("Creating CC sector %d", sid)
-	return sectorID, m.sectors.Send(uint64(sid), SectorStartCC{
+	return m.minerSector(spt, sid), m.sectors.Send(uint64(sid), SectorStartCC{
 		ID:         sid,
 		SectorType: spt,
 	})

extern/storage-sealing/input.go

@@ -16,6 +16,7 @@ import (
 
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
 func (m *Sealing) handleWaitDeals(ctx statemachine.Context, sector SectorInfo) error {
@@ -388,16 +389,9 @@ func (m *Sealing) tryCreateDealSector(ctx context.Context, sp abi.RegisteredSeal
 		return nil
 	}
 
-	// Now actually create a new sector
+	sid, err := m.createSector(ctx, cfg, sp)
-
-	sid, err := m.sc.Next()
 	if err != nil {
-		return xerrors.Errorf("getting sector number: %w", err)
+		return err
-	}
-
-	err = m.sealer.NewSector(ctx, m.minerSector(sp, sid))
-	if err != nil {
-		return xerrors.Errorf("initializing sector: %w", err)
 	}
 
 	log.Infow("Creating sector", "number", sid, "type", "deal", "proofType", sp)
@@ -407,6 +401,26 @@ func (m *Sealing) tryCreateDealSector(ctx context.Context, sp abi.RegisteredSeal
 	})
 }
 
+// call with m.inputLk
+func (m *Sealing) createSector(ctx context.Context, cfg sealiface.Config, sp abi.RegisteredSealProof) (abi.SectorNumber, error) {
+	// Now actually create a new sector
+
+	sid, err := m.sc.Next()
+	if err != nil {
+		return 0, xerrors.Errorf("getting sector number: %w", err)
+	}
+
+	err = m.sealer.NewSector(ctx, m.minerSector(sp, sid))
+	if err != nil {
+		return 0, xerrors.Errorf("initializing sector: %w", err)
+	}
+
+	// update stats early, fsm planner would do that async
+	m.stats.updateSector(cfg, m.minerSectorID(sid), UndefinedSectorState)
+
+	return sid, nil
+}
+
 func (m *Sealing) StartPacking(sid abi.SectorNumber) error {
 	return m.sectors.Send(uint64(sid), SectorStartPacking{})
 }

extern/storage-sealing/stats.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
 type statSectorState int
@@ -23,10 +24,14 @@ type SectorStats struct {
 	totals   [nsst]uint64
 }
 
-func (ss *SectorStats) updateSector(id abi.SectorID, st SectorState) {
+func (ss *SectorStats) updateSector(cfg sealiface.Config, id abi.SectorID, st SectorState) (updateInput bool) {
 	ss.lk.Lock()
 	defer ss.lk.Unlock()
 
+	preSealing := ss.curSealingLocked()
+	preStaging := ss.curStagingLocked()
+
+	// update totals
 	oldst, found := ss.bySector[id]
 	if found {
 		ss.totals[oldst]--
@@ -35,6 +40,34 @@ func (ss *SectorStats) updateSector(id abi.SectorID, st SectorState) {
 	sst := toStatState(st)
 	ss.bySector[id] = sst
 	ss.totals[sst]++
+
+	// check if we may need be able to process more deals
+	sealing := ss.curSealingLocked()
+	staging := ss.curStagingLocked()
+
+	log.Debugw("sector stats", "sealing", sealing, "staging", staging)
+
+	if cfg.MaxSealingSectorsForDeals > 0 && // max sealing deal sector limit set
+		preSealing >= cfg.MaxSealingSectorsForDeals && // we were over limit
+		sealing < cfg.MaxSealingSectorsForDeals { // and we're below the limit now
+		updateInput = true
+	}
+
+	if cfg.MaxWaitDealsSectors > 0 && // max waiting deal sector limit set
+		preStaging >= cfg.MaxWaitDealsSectors && // we were over limit
+		staging < cfg.MaxWaitDealsSectors { // and we're below the limit now
+		updateInput = true
+	}
+
+	return updateInput
+}
+
+func (ss *SectorStats) curSealingLocked() uint64 {
+	return ss.totals[sstStaging] + ss.totals[sstSealing] + ss.totals[sstFailed]
+}
+
+func (ss *SectorStats) curStagingLocked() uint64 {
+	return ss.totals[sstStaging]
 }
 
 // return the number of sectors currently in the sealing pipeline
@@ -42,7 +75,7 @@ func (ss *SectorStats) curSealing() uint64 {
 	ss.lk.Lock()
 	defer ss.lk.Unlock()
 
-	return ss.totals[sstStaging] + ss.totals[sstSealing] + ss.totals[sstFailed]
+	return ss.curSealingLocked()
 }
 
 // return the number of sectors waiting to enter the sealing pipeline
@@ -50,5 +83,5 @@ func (ss *SectorStats) curStaging() uint64 {
 	ss.lk.Lock()
 	defer ss.lk.Unlock()
 
-	return ss.totals[sstStaging]
+	return ss.curStagingLocked()
 }

gen/api/proxygen.go

@@ -0,0 +1,297 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"unicode"
+
+	"golang.org/x/xerrors"
+)
+
+type methodMeta struct {
+	node  ast.Node
+	ftype *ast.FuncType
+}
+
+type Visitor struct {
+	Methods map[string]map[string]*methodMeta
+	Include map[string][]string
+}
+
+func (v *Visitor) Visit(node ast.Node) ast.Visitor {
+	st, ok := node.(*ast.TypeSpec)
+	if !ok {
+		return v
+	}
+
+	iface, ok := st.Type.(*ast.InterfaceType)
+	if !ok {
+		return v
+	}
+	if v.Methods[st.Name.Name] == nil {
+		v.Methods[st.Name.Name] = map[string]*methodMeta{}
+	}
+	for _, m := range iface.Methods.List {
+		switch ft := m.Type.(type) {
+		case *ast.Ident:
+			v.Include[st.Name.Name] = append(v.Include[st.Name.Name], ft.Name)
+		case *ast.FuncType:
+			v.Methods[st.Name.Name][m.Names[0].Name] = &methodMeta{
+				node:  m,
+				ftype: ft,
+			}
+		}
+	}
+
+	return v
+}
+func main() {
+	if err := runMain(); err != nil {
+		fmt.Println("error: ", err)
+	}
+}
+
+func typeName(e ast.Expr) (string, error) {
+	switch t := e.(type) {
+	case *ast.SelectorExpr:
+		return t.X.(*ast.Ident).Name + "." + t.Sel.Name, nil
+	case *ast.Ident:
+		pstr := t.Name
+		if !unicode.IsLower(rune(pstr[0])) {
+			pstr = "api." + pstr // todo src pkg name
+		}
+		return pstr, nil
+	case *ast.ArrayType:
+		subt, err := typeName(t.Elt)
+		if err != nil {
+			return "", err
+		}
+		return "[]" + subt, nil
+	case *ast.StarExpr:
+		subt, err := typeName(t.X)
+		if err != nil {
+			return "", err
+		}
+		return "*" + subt, nil
+	case *ast.MapType:
+		k, err := typeName(t.Key)
+		if err != nil {
+			return "", err
+		}
+		v, err := typeName(t.Value)
+		if err != nil {
+			return "", err
+		}
+		return "map[" + k + "]" + v, nil
+	case *ast.StructType:
+		if len(t.Fields.List) != 0 {
+			return "", xerrors.Errorf("can't struct")
+		}
+		return "struct{}", nil
+	case *ast.InterfaceType:
+		if len(t.Methods.List) != 0 {
+			return "", xerrors.Errorf("can't interface")
+		}
+		return "interface{}", nil
+	case *ast.ChanType:
+		subt, err := typeName(t.Value)
+		if err != nil {
+			return "", err
+		}
+		if t.Dir == ast.SEND {
+			subt = "->chan " + subt
+		} else {
+			subt = "<-chan " + subt
+		}
+		return subt, nil
+	default:
+		return "", xerrors.Errorf("unknown type")
+	}
+}
+
+func runMain() error {
+	fset := token.NewFileSet()
+	apiDir, err := filepath.Abs("./api")
+	if err != nil {
+		return err
+	}
+	pkgs, err := parser.ParseDir(fset, apiDir, nil, parser.AllErrors|parser.ParseComments)
+	if err != nil {
+		return err
+	}
+
+	ap := pkgs["api"]
+
+	v := &Visitor{make(map[string]map[string]*methodMeta), map[string][]string{}}
+	ast.Walk(v, ap)
+
+	type methodInfo struct {
+		Name                             string
+		node                             ast.Node
+		Tags                             map[string][]string
+		NamedParams, ParamNames, Results string
+	}
+
+	type strinfo struct {
+		Name    string
+		Methods map[string]*methodInfo
+		Include []string
+	}
+
+	type meta struct {
+		Infos   map[string]*strinfo
+		Imports map[string]string
+	}
+
+	m := &meta{
+		Infos:   map[string]*strinfo{},
+		Imports: map[string]string{},
+	}
+
+	for fn, f := range ap.Files {
+		if strings.HasSuffix(fn, "gen.go") {
+			continue
+		}
+
+		//fmt.Println("F:", fn)
+		cmap := ast.NewCommentMap(fset, f, f.Comments)
+
+		for _, im := range f.Imports {
+			m.Imports[im.Path.Value] = im.Path.Value
+		}
+
+		for ifname, methods := range v.Methods {
+			if _, ok := m.Infos[ifname]; !ok {
+				m.Infos[ifname] = &strinfo{
+					Name:    ifname,
+					Methods: map[string]*methodInfo{},
+					Include: v.Include[ifname],
+				}
+			}
+			info := m.Infos[ifname]
+			for mname, node := range methods {
+				filteredComments := cmap.Filter(node.node).Comments()
+
+				if _, ok := info.Methods[mname]; !ok {
+					var params, pnames []string
+					for _, param := range node.ftype.Params.List {
+						pstr, err := typeName(param.Type)
+						if err != nil {
+							return err
+						}
+
+						c := len(param.Names)
+						if c == 0 {
+							c = 1
+						}
+
+						for i := 0; i < c; i++ {
+							pname := fmt.Sprintf("p%d", len(params))
+							pnames = append(pnames, pname)
+							params = append(params, pname+" "+pstr)
+						}
+					}
+
+					var results []string
+					for _, result := range node.ftype.Results.List {
+						rs, err := typeName(result.Type)
+						if err != nil {
+							return err
+						}
+						results = append(results, rs)
+					}
+
+					info.Methods[mname] = &methodInfo{
+						Name:        mname,
+						node:        node.node,
+						Tags:        map[string][]string{},
+						NamedParams: strings.Join(params, ", "),
+						ParamNames:  strings.Join(pnames, ", "),
+						Results:     strings.Join(results, ", "),
+					}
+				}
+
+				// try to parse tag info
+				if len(filteredComments) > 0 {
+					tagstr := filteredComments[len(filteredComments)-1].List[0].Text
+					tagstr = strings.TrimPrefix(tagstr, "//")
+					tl := strings.Split(strings.TrimSpace(tagstr), " ")
+					for _, ts := range tl {
+						tf := strings.Split(ts, ":")
+						if len(tf) != 2 {
+							continue
+						}
+						if tf[0] != "perm" { // todo: allow more tag types
+							continue
+						}
+						info.Methods[mname].Tags[tf[0]] = tf
+					}
+				}
+			}
+		}
+	}
+
+	/*jb, err := json.MarshalIndent(Infos, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(jb))*/
+
+	w := os.Stdout
+
+	err = doTemplate(w, m, `// Code generated by github.com/filecoin-project/lotus/gen/api. DO NOT EDIT.
+
+package apistruct
+
+import (
+{{range .Imports}}{{.}}
+{{end}}
+)
+`)
+	if err != nil {
+		return err
+	}
+
+	err = doTemplate(w, m, `
+{{range .Infos}}
+type {{.Name}}Struct struct {
+{{range .Include}}
+	{{.}}Struct
+{{end}}
+	Internal struct {
+{{range .Methods}}
+		{{.Name}} func({{.NamedParams}}) ({{.Results}}) `+"`"+`{{range .Tags}}{{index . 0}}:"{{index . 1}}"{{end}}`+"`"+`
+{{end}}
+	}
+}
+{{end}}
+
+{{range .Infos}}
+{{$name := .Name}}
+{{range .Methods}}
+func (s *{{$name}}Struct) {{.Name}}({{.NamedParams}}) ({{.Results}}) {
+	return s.Internal.{{.Name}}({{.ParamNames}})
+}
+{{end}}
+{{end}}
+
+{{range .Infos}}var _ api.{{.Name}} = new({{.Name}}Struct)
+{{end}}
+
+`)
+	return err
+}
+
+func doTemplate(w io.Writer, info interface{}, templ string) error {
+	t := template.Must(template.New("").
+		Funcs(template.FuncMap{}).Parse(templ))
+
+	return t.Execute(w, info)
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
+	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07
@@ -22,6 +23,7 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elastic/go-sysinfo v1.3.0
 	github.com/elastic/gosigar v0.12.0
+	github.com/etclabscore/go-openrpc-reflect v0.0.36
 	github.com/fatih/color v1.9.0
 	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f
 	github.com/filecoin-project/go-address v0.0.5
@@ -122,11 +124,13 @@ require (
 	github.com/multiformats/go-multibase v0.0.3
 	github.com/multiformats/go-multihash v0.0.14
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
 	github.com/prometheus/client_golang v1.6.0
 	github.com/raulk/clock v1.1.0
 	github.com/raulk/go-watchdog v1.0.1
+	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/urfave/cli/v2 v2.2.0
@@ -142,7 +146,7 @@ require (
 	go.uber.org/fx v1.9.0
 	go.uber.org/multierr v1.6.0
 	go.uber.org/zap v1.16.0
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974
+	golang.org/x/net v0.0.0-20201022231255-08b38378de70
 	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
@@ -150,6 +154,7 @@ require (
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools v2.2.0+incompatible
+	honnef.co/go/tools v0.0.1-2020.1.3 // indirect
 )
 
 replace github.com/filecoin-project/lotus => ./

go.sum

@@ -42,8 +42,15 @@ github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K1
 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@@ -58,6 +65,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o=
+github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -140,6 +149,7 @@ github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbL
 github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327 h1:7grrpcfCtbZLsjtB0DgMuzs1umsJmpzaHMZ6cO6iAWw=
 github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -155,6 +165,7 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7
 github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg=
 github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -223,6 +234,10 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakRE22OTI12k+2LkyY=
+github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ=
+github.com/etclabscore/go-openrpc-reflect v0.0.36 h1:kSqNB2U8RVoW4si+4fsv13NGNkRAQ5j78zTUx1qiehk=
+github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -333,6 +348,21 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.4 h1:3Vw+rh13uq2JFNxgnMTGE1rnoieU9FmyE1gvnyylsYg=
+github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.11 h1:ogU5q8dtp3MMPn59a9VRrPKVxvJHEs5P7yNMR5sNnis=
+github.com/go-openapi/spec v0.19.11/go.mod h1:vqK/dIdLGCosfvYsQV3WfC7N3TiZSnGY2RZKoFK7X28=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
+github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc=
+github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@@ -366,6 +396,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -484,7 +515,11 @@ github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOo
 github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
 github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
+github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3F6cCkg=
+github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
@@ -710,6 +745,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1 h1:qBCV/RLV02TSfQa7tFmxTihnG+u+7JXByOkhlkR5rmQ=
 github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
@@ -1078,6 +1115,11 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
@@ -1212,6 +1254,7 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA=
@@ -1239,6 +1282,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0=
+github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI=
 github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc=
@@ -1406,10 +1451,12 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -1420,6 +1467,12 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
+github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
+github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -1664,13 +1717,15 @@ golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200519113804-d87ec0cfa476/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201022231255-08b38378de70 h1:Z6x4N9mAi4oF0TbHweCsH618MO6OI6UFgV0FP5n0wBY=
+golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1798,6 +1853,7 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -1928,8 +1984,9 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
 modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ=

lib/backupds/backupds_test.go

@@ -0,0 +1,85 @@
+package backupds
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+	"github.com/stretchr/testify/require"
+)
+
+const valSize = 512 << 10
+
+func putVals(t *testing.T, ds datastore.Datastore, start, end int) {
+	for i := start; i < end; i++ {
+		err := ds.Put(datastore.NewKey(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%d-%s", i, strings.Repeat("~", valSize))))
+		require.NoError(t, err)
+	}
+}
+
+func checkVals(t *testing.T, ds datastore.Datastore, start, end int, exist bool) {
+	for i := start; i < end; i++ {
+		v, err := ds.Get(datastore.NewKey(fmt.Sprintf("%d", i)))
+		if exist {
+			require.NoError(t, err)
+			expect := []byte(fmt.Sprintf("%d-%s", i, strings.Repeat("~", valSize)))
+			require.EqualValues(t, expect, v)
+		} else {
+			require.ErrorIs(t, err, datastore.ErrNotFound)
+		}
+	}
+}
+
+func TestNoLogRestore(t *testing.T) {
+	ds1 := datastore.NewMapDatastore()
+
+	putVals(t, ds1, 0, 10)
+
+	bds, err := Wrap(ds1, NoLogdir)
+	require.NoError(t, err)
+
+	var bup bytes.Buffer
+	require.NoError(t, bds.Backup(&bup))
+
+	putVals(t, ds1, 10, 20)
+
+	ds2 := datastore.NewMapDatastore()
+	require.NoError(t, RestoreInto(&bup, ds2))
+
+	checkVals(t, ds2, 0, 10, true)
+	checkVals(t, ds2, 10, 20, false)
+}
+
+func TestLogRestore(t *testing.T) {
+	logdir, err := ioutil.TempDir("", "backupds-test-")
+	require.NoError(t, err)
+	defer os.RemoveAll(logdir) // nolint
+
+	ds1 := datastore.NewMapDatastore()
+
+	putVals(t, ds1, 0, 10)
+
+	bds, err := Wrap(ds1, logdir)
+	require.NoError(t, err)
+
+	putVals(t, bds, 10, 20)
+
+	require.NoError(t, bds.Close())
+
+	fls, err := ioutil.ReadDir(logdir)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(fls))
+
+	bf, err := ioutil.ReadFile(filepath.Join(logdir, fls[0].Name()))
+	require.NoError(t, err)
+
+	ds2 := datastore.NewMapDatastore()
+	require.NoError(t, RestoreInto(bytes.NewReader(bf), ds2))
+
+	checkVals(t, ds2, 0, 20, true)
+}

markets/retrievaladapter/provider.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"io"
 
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -16,10 +19,10 @@ import (
 	"github.com/filecoin-project/go-fil-markets/shared"
 	"github.com/filecoin-project/go-state-types/abi"
 	specstorage "github.com/filecoin-project/specs-storage/storage"
-
-	"github.com/ipfs/go-cid"
 )
 
+var log = logging.Logger("retrievaladapter")
+
 type retrievalProviderNode struct {
 	miner  *storage.Miner
 	sealer sectorstorage.SectorManager
@@ -61,13 +64,20 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi
 		ProofType: si.SectorType,
 	}
 
+	// Set up a pipe so that data can be written from the unsealing process
+	// into the reader returned by this function
 	r, w := io.Pipe()
 	go func() {
 		var commD cid.Cid
 		if si.CommD != nil {
 			commD = *si.CommD
 		}
+		// Read the piece into the pipe's writer, unsealing the piece if necessary
 		err := rpn.sealer.ReadPiece(ctx, w, ref, storiface.UnpaddedByteIndex(offset), length, si.TicketValue, commD)
+		if err != nil {
+			log.Errorf("failed to unseal piece from sector %d: %s", sectorID, err)
+		}
+		// Close the reader with any error that was returned while reading the piece
 		_ = w.CloseWithError(err)
 	}()
 

metrics/proxy.go

@@ -24,19 +24,19 @@ func MetricedFullAPI(a api.FullNode) api.FullNode {
 	return &out
 }
 
-func MetricedWorkerAPI(a api.WorkerAPI) api.WorkerAPI {
+func MetricedWorkerAPI(a api.Worker) api.Worker {
 	var out apistruct.WorkerStruct
 	proxy(a, &out.Internal)
 	return &out
 }
 
-func MetricedWalletAPI(a api.WalletAPI) api.WalletAPI {
+func MetricedWalletAPI(a api.Wallet) api.Wallet {
 	var out apistruct.WalletStruct
 	proxy(a, &out.Internal)
 	return &out
 }
 
-func MetricedGatewayAPI(a api.GatewayAPI) api.GatewayAPI {
+func MetricedGatewayAPI(a api.Gateway) api.Gateway {
 	var out apistruct.GatewayStruct
 	proxy(a, &out.Internal)
 	return &out

node/builder.go

@@ -11,6 +11,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/exchange"
+	rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/chain/wallet"
@@ -304,12 +305,13 @@ var ChainNode = Options(
 	Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
 	Override(new(*wallet.LocalWallet), wallet.NewWallet),
 	Override(new(wallet.Default), From(new(*wallet.LocalWallet))),
-	Override(new(api.WalletAPI), From(new(wallet.MultiWallet))),
+	Override(new(api.Wallet), From(new(wallet.MultiWallet))),
 
 	// Service: Payment channels
-	Override(new(*paychmgr.Store), paychmgr.NewStore),
+	Override(new(paychmgr.PaychAPI), From(new(modules.PaychAPI))),
-	Override(new(*paychmgr.Manager), paychmgr.NewManager),
+	Override(new(*paychmgr.Store), modules.NewPaychStore),
-	Override(HandlePaymentChannelManagerKey, paychmgr.HandleManager),
+	Override(new(*paychmgr.Manager), modules.NewManager),
+	Override(HandlePaymentChannelManagerKey, modules.HandlePaychManager),
 	Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels),
 
 	// Markets (common)
@@ -327,14 +329,16 @@ var ChainNode = Options(
 	Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
 	Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds),
 
+	Override(new(*full.GasPriceCache), full.NewGasPriceCache),
+
 	// Lite node API
 	ApplyIf(isLiteNode,
 		Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))),
-		Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))),
+		Override(new(full.ChainModuleAPI), From(new(api.Gateway))),
-		Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))),
+		Override(new(full.GasModuleAPI), From(new(api.Gateway))),
-		Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))),
+		Override(new(full.MpoolModuleAPI), From(new(api.Gateway))),
-		Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))),
+		Override(new(full.StateModuleAPI), From(new(api.Gateway))),
-		Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager),
+		Override(new(stmgr.StateManagerAPI), rpcstmgr.NewRPCStateManager),
 	),
 
 	// Full node API / service startup
@@ -725,3 +729,19 @@ func Test() Option {
 		Override(new(*storageadapter.DealPublisher), storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{})),
 	)
 }
+
+// For 3rd party dep injection.
+
+func WithRepoType(repoType repo.RepoType) func(s *Settings) error {
+	return func(s *Settings) error {
+		s.nodeType = repoType
+		return nil
+	}
+}
+
+func WithInvokesKey(i invoke, resApi interface{}) func(s *Settings) error {
+	return func(s *Settings) error {
+		s.invokes[i] = fx.Populate(resApi)
+		return nil
+	}
+}

node/config/def.go

@@ -236,6 +236,7 @@ func DefaultStorageMiner() *StorageMiner {
 			MaxSealingSectors:         0,
 			MaxSealingSectorsForDeals: 0,
 			WaitDealsDelay:            Duration(time.Hour * 6),
+			AlwaysKeepUnsealedCopy:    true,
 		},
 
 		Storage: sectorstorage.SealerConfig{

node/impl/common/common.go

@@ -24,6 +24,7 @@ import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
 	"github.com/filecoin-project/lotus/api"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/lp2p"
@@ -207,6 +208,10 @@ func (a *CommonAPI) NetBandwidthStatsByProtocol(ctx context.Context) (map[protoc
 	return a.Reporter.GetBandwidthByProtocol(), nil
 }
 
+func (a *CommonAPI) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
+	return build.OpenRPCDiscoverJSON_Full(), nil
+}
+
 func (a *CommonAPI) ID(context.Context) (peer.ID, error) {
 	return a.Host.ID(), nil
 }

node/impl/full/gas.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
+	lru "github.com/hashicorp/golang-lru"
 
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
@@ -39,6 +40,8 @@ type GasModule struct {
 	Chain     *store.ChainStore
 	Mpool     *messagepool.MessagePool
 	GetMaxFee dtypes.DefaultMaxFeeFunc
+
+	PriceCache *GasPriceCache
 }
 
 var _ GasModuleAPI = (*GasModule)(nil)
@@ -51,6 +54,53 @@ type GasAPI struct {
 	Stmgr *stmgr.StateManager
 	Chain *store.ChainStore
 	Mpool *messagepool.MessagePool
+
+	PriceCache *GasPriceCache
+}
+
+func NewGasPriceCache() *GasPriceCache {
+	// 50 because we usually won't access more than 40
+	c, err := lru.New2Q(50)
+	if err != nil {
+		// err only if parameter is bad
+		panic(err)
+	}
+
+	return &GasPriceCache{
+		c: c,
+	}
+}
+
+type GasPriceCache struct {
+	c *lru.TwoQueueCache
+}
+
+type GasMeta struct {
+	Price big.Int
+	Limit int64
+}
+
+func (g *GasPriceCache) GetTSGasStats(cstore *store.ChainStore, ts *types.TipSet) ([]GasMeta, error) {
+	i, has := g.c.Get(ts.Key())
+	if has {
+		return i.([]GasMeta), nil
+	}
+
+	var prices []GasMeta
+	msgs, err := cstore.MessagesForTipset(ts)
+	if err != nil {
+		return nil, xerrors.Errorf("loading messages: %w", err)
+	}
+	for _, msg := range msgs {
+		prices = append(prices, GasMeta{
+			Price: msg.VMMessage().GasPremium,
+			Limit: msg.VMMessage().GasLimit,
+		})
+	}
+
+	g.c.Add(ts.Key(), prices)
+
+	return prices, nil
 }
 
 const MinGasPremium = 100e3
@@ -88,24 +138,19 @@ func gasEstimateFeeCap(cstore *store.ChainStore, msg *types.Message, maxqueueblk
 	return out, nil
 }
 
-type gasMeta struct {
-	price big.Int
-	limit int64
-}
-
 // finds 55th percntile instead of median to put negative pressure on gas price
-func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount {
+func medianGasPremium(prices []GasMeta, blocks int) abi.TokenAmount {
 	sort.Slice(prices, func(i, j int) bool {
 		// sort desc by price
-		return prices[i].price.GreaterThan(prices[j].price)
+		return prices[i].Price.GreaterThan(prices[j].Price)
 	})
 
 	at := build.BlockGasTarget * int64(blocks) / 2        // 50th
 	at += build.BlockGasTarget * int64(blocks) / (2 * 20) // move 5% further
 	prev1, prev2 := big.Zero(), big.Zero()
 	for _, price := range prices {
-		prev1, prev2 = price.price, prev1
+		prev1, prev2 = price.Price, prev1
-		at -= price.limit
+		at -= price.Limit
 		if at < 0 {
 			break
 		}
@@ -126,7 +171,7 @@ func (a *GasAPI) GasEstimateGasPremium(
 	gaslimit int64,
 	_ types.TipSetKey,
 ) (types.BigInt, error) {
-	return gasEstimateGasPremium(a.Chain, nblocksincl)
+	return gasEstimateGasPremium(a.Chain, a.PriceCache, nblocksincl)
 }
 func (m *GasModule) GasEstimateGasPremium(
 	ctx context.Context,
@@ -135,14 +180,14 @@ func (m *GasModule) GasEstimateGasPremium(
 	gaslimit int64,
 	_ types.TipSetKey,
 ) (types.BigInt, error) {
-	return gasEstimateGasPremium(m.Chain, nblocksincl)
+	return gasEstimateGasPremium(m.Chain, m.PriceCache, nblocksincl)
 }
-func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.BigInt, error) {
+func gasEstimateGasPremium(cstore *store.ChainStore, cache *GasPriceCache, nblocksincl uint64) (types.BigInt, error) {
 	if nblocksincl == 0 {
 		nblocksincl = 1
 	}
 
-	var prices []gasMeta
+	var prices []GasMeta
 	var blocks int
 
 	ts := cstore.GetHeaviestTipSet()
@@ -157,17 +202,11 @@ func gasEstimateGasPremium(cstore *store.ChainStore, nblocksincl uint64) (types.
 		}
 
 		blocks += len(pts.Blocks())
-
+		meta, err := cache.GetTSGasStats(cstore, pts)
-		msgs, err := cstore.MessagesForTipset(pts)
 		if err != nil {
-			return types.BigInt{}, xerrors.Errorf("loading messages: %w", err)
+			return types.BigInt{}, err
-		}
-		for _, msg := range msgs {
-			prices = append(prices, gasMeta{
-				price: msg.VMMessage().GasPremium,
-				limit: msg.VMMessage().GasLimit,
-			})
 		}
+		prices = append(prices, meta...)
 
 		ts = pts
 	}

node/impl/full/gas_test.go

@@ -12,27 +12,27 @@ import (
 )
 
 func TestMedian(t *testing.T) {
-	require.Equal(t, types.NewInt(5), medianGasPremium([]gasMeta{
+	require.Equal(t, types.NewInt(5), medianGasPremium([]GasMeta{
 		{big.NewInt(5), build.BlockGasTarget},
 	}, 1))
 
-	require.Equal(t, types.NewInt(10), medianGasPremium([]gasMeta{
+	require.Equal(t, types.NewInt(10), medianGasPremium([]GasMeta{
 		{big.NewInt(5), build.BlockGasTarget},
 		{big.NewInt(10), build.BlockGasTarget},
 	}, 1))
 
-	require.Equal(t, types.NewInt(15), medianGasPremium([]gasMeta{
+	require.Equal(t, types.NewInt(15), medianGasPremium([]GasMeta{
 		{big.NewInt(10), build.BlockGasTarget / 2},
 		{big.NewInt(20), build.BlockGasTarget / 2},
 	}, 1))
 
-	require.Equal(t, types.NewInt(25), medianGasPremium([]gasMeta{
+	require.Equal(t, types.NewInt(25), medianGasPremium([]GasMeta{
 		{big.NewInt(10), build.BlockGasTarget / 2},
 		{big.NewInt(20), build.BlockGasTarget / 2},
 		{big.NewInt(30), build.BlockGasTarget / 2},
 	}, 1))
 
-	require.Equal(t, types.NewInt(15), medianGasPremium([]gasMeta{
+	require.Equal(t, types.NewInt(15), medianGasPremium([]GasMeta{
 		{big.NewInt(10), build.BlockGasTarget / 2},
 		{big.NewInt(20), build.BlockGasTarget / 2},
 		{big.NewInt(30), build.BlockGasTarget / 2},

node/impl/full/state.go

@@ -76,7 +76,7 @@ type StateAPI struct {
 
 	// TODO: the wallet here is only needed because we have the MinerCreateBlock
 	// API attached to the state API. It probably should live somewhere better
-	Wallet    api.WalletAPI
+	Wallet    api.Wallet
 	DefWallet wallet.Default
 
 	StateModuleAPI

node/impl/full/wallet.go

@@ -22,7 +22,7 @@ type WalletAPI struct {
 
 	StateManagerAPI stmgr.StateManagerAPI
 	Default         wallet.Default
-	api.WalletAPI
+	api.Wallet
 }
 
 func (a *WalletAPI) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) {
@@ -40,7 +40,7 @@ func (a *WalletAPI) WalletSign(ctx context.Context, k address.Address, msg []byt
 	if err != nil {
 		return nil, xerrors.Errorf("failed to resolve ID address: %w", keyAddr)
 	}
-	return a.WalletAPI.WalletSign(ctx, keyAddr, msg, api.MsgMeta{
+	return a.Wallet.WalletSign(ctx, keyAddr, msg, api.MsgMeta{
 		Type: api.MTUnknown,
 	})
 }
@@ -56,7 +56,7 @@ func (a *WalletAPI) WalletSignMessage(ctx context.Context, k address.Address, ms
 		return nil, xerrors.Errorf("serializing message: %w", err)
 	}
 
-	sig, err := a.WalletAPI.WalletSign(ctx, keyAddr, mb.Cid().Bytes(), api.MsgMeta{
+	sig, err := a.Wallet.WalletSign(ctx, keyAddr, mb.Cid().Bytes(), api.MsgMeta{
 		Type:  api.MTChainMsg,
 		Extra: mb.RawData(),
 	})

node/impl/remoteworker.go

@@ -16,7 +16,7 @@ import (
 )
 
 type remoteWorker struct {
-	api.WorkerAPI
+	api.Worker
 	closer jsonrpc.ClientCloser
 }
 

node/impl/storminer.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/host"
@@ -31,6 +32,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/markets/storageadapter"
 	"github.com/filecoin-project/lotus/miner"
@@ -690,4 +692,8 @@ func (sm *StorageMinerAPI) ActorAddressConfig(ctx context.Context) (api.AddressC
 	return sm.AddrSel.AddressConfig, nil
 }
 
+func (sm *StorageMinerAPI) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
+	return build.OpenRPCDiscoverJSON_Miner(), nil
+}
+
 var _ api.StorageMiner = &StorageMinerAPI{}

node/modules/paych.go

@@ -0,0 +1,45 @@
+package modules
+
+import (
+	"context"
+
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/node/impl/full"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/modules/helpers"
+	"github.com/filecoin-project/lotus/paychmgr"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	"go.uber.org/fx"
+)
+
+func NewManager(mctx helpers.MetricsCtx, lc fx.Lifecycle, sm stmgr.StateManagerAPI, pchstore *paychmgr.Store, api paychmgr.PaychAPI) *paychmgr.Manager {
+	ctx := helpers.LifecycleCtx(mctx, lc)
+	ctx, shutdown := context.WithCancel(ctx)
+
+	return paychmgr.NewManager(ctx, shutdown, sm, pchstore, api)
+}
+
+func NewPaychStore(ds dtypes.MetadataDS) *paychmgr.Store {
+	ds = namespace.Wrap(ds, datastore.NewKey("/paych/"))
+	return paychmgr.NewStore(ds)
+}
+
+type PaychAPI struct {
+	fx.In
+
+	full.MpoolAPI
+	full.StateAPI
+}
+
+// HandlePaychManager is called by dependency injection to set up hooks
+func HandlePaychManager(lc fx.Lifecycle, pm *paychmgr.Manager) {
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			return pm.Start()
+		},
+		OnStop: func(context.Context) error {
+			return pm.Stop()
+		},
+	})
+}

node/node_test.go

@@ -58,6 +58,9 @@ func TestAPIDealFlow(t *testing.T) {
 	t.Run("TestPublishDealsBatching", func(t *testing.T) {
 		test.TestPublishDealsBatching(t, builder.MockSbBuilder, blockTime, dealStartEpoch)
 	})
+	t.Run("TestBatchDealInput", func(t *testing.T) {
+		test.TestBatchDealInput(t, builder.MockSbBuilder, blockTime, dealStartEpoch)
+	})
 }
 
 func TestAPIDealFlowReal(t *testing.T) {

node/repo/fsrepo.go

@@ -545,17 +545,31 @@ func (fsr *fsLockedRepo) Get(name string) (types.KeyInfo, error) {
 	return res, nil
 }
 
+const KTrashPrefix = "trash-"
+
 // Put saves key info under given name
 func (fsr *fsLockedRepo) Put(name string, info types.KeyInfo) error {
+	return fsr.put(name, info, 0)
+}
+
+func (fsr *fsLockedRepo) put(rawName string, info types.KeyInfo, retries int) error {
 	if err := fsr.stillValid(); err != nil {
 		return err
 	}
 
+	name := rawName
+	if retries > 0 {
+		name = fmt.Sprintf("%s-%d", rawName, retries)
+	}
+
 	encName := base32.RawStdEncoding.EncodeToString([]byte(name))
 	keyPath := fsr.join(fsKeystore, encName)
 
 	_, err := os.Stat(keyPath)
-	if err == nil {
+	if err == nil && strings.HasPrefix(name, KTrashPrefix) {
+		// retry writing the trash-prefixed file with a number suffix
+		return fsr.put(rawName, info, retries+1)
+	} else if err == nil {
 		return xerrors.Errorf("checking key before put '%s': %w", name, types.ErrKeyExists)
 	} else if !os.IsNotExist(err) {
 		return xerrors.Errorf("checking key before put '%s': %w", name, err)

paychmgr/manager.go

@@ -8,7 +8,6 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
-	"go.uber.org/fx"
 	xerrors "golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -19,22 +18,12 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node/impl/full"
-	"github.com/filecoin-project/lotus/node/modules/helpers"
 )
 
 var log = logging.Logger("paych")
 
 var errProofNotSupported = errors.New("payment channel proof parameter is not supported")
 
-// PaychAPI is used by dependency injection to pass the consituent APIs to NewManager()
-type PaychAPI struct {
-	fx.In
-
-	full.MpoolAPI
-	full.StateAPI
-}
-
 // stateManagerAPI defines the methods needed from StateManager
 type stateManagerAPI interface {
 	ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error)
@@ -43,7 +32,7 @@ type stateManagerAPI interface {
 }
 
 // paychAPI defines the API methods needed by the payment channel manager
-type paychAPI interface {
+type PaychAPI interface {
 	StateAccountKey(context.Context, address.Address, types.TipSetKey) (address.Address, error)
 	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error)
 	MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error)
@@ -55,13 +44,13 @@ type paychAPI interface {
 // managerAPI defines all methods needed by the manager
 type managerAPI interface {
 	stateManagerAPI
-	paychAPI
+	PaychAPI
 }
 
 // managerAPIImpl is used to create a composite that implements managerAPI
 type managerAPIImpl struct {
 	stmgr.StateManagerAPI
-	paychAPI
+	PaychAPI
 }
 
 type Manager struct {
@@ -77,11 +66,8 @@ type Manager struct {
 	channels map[string]*channelAccessor
 }
 
-func NewManager(mctx helpers.MetricsCtx, lc fx.Lifecycle, sm stmgr.StateManagerAPI, pchstore *Store, api PaychAPI) *Manager {
+func NewManager(ctx context.Context, shutdown func(), sm stmgr.StateManagerAPI, pchstore *Store, api PaychAPI) *Manager {
-	ctx := helpers.LifecycleCtx(mctx, lc)
+	impl := &managerAPIImpl{StateManagerAPI: sm, PaychAPI: api}
-	ctx, shutdown := context.WithCancel(ctx)
-
-	impl := &managerAPIImpl{StateManagerAPI: sm, paychAPI: &api}
 	return &Manager{
 		ctx:      ctx,
 		shutdown: shutdown,
@@ -103,18 +89,6 @@ func newManager(pchstore *Store, pchapi managerAPI) (*Manager, error) {
 	return pm, pm.Start()
 }
 
-// HandleManager is called by dependency injection to set up hooks
-func HandleManager(lc fx.Lifecycle, pm *Manager) {
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			return pm.Start()
-		},
-		OnStop: func(context.Context) error {
-			return pm.Stop()
-		},
-	})
-}
-
 // Start restarts tracking of any messages that were sent to chain.
 func (pm *Manager) Start() error {
 	return pm.restartPending()

paychmgr/store.go

@@ -14,14 +14,12 @@ import (
 	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
 	dsq "github.com/ipfs/go-datastore/query"
 
 	"github.com/filecoin-project/go-address"
 	cborrpc "github.com/filecoin-project/go-cbor-util"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
 var ErrChannelNotTracked = errors.New("channel not tracked")
@@ -30,8 +28,7 @@ type Store struct {
 	ds datastore.Batching
 }
 
-func NewStore(ds dtypes.MetadataDS) *Store {
+func NewStore(ds datastore.Batching) *Store {
-	ds = namespace.Wrap(ds, datastore.NewKey("/paych/"))
 	return &Store{
 		ds: ds,
 	}