Merge pull request #5390 from filecoin-project/asr/changelog

Lotus version 1.4.1

.circleci/config.yml

@@ -200,6 +200,8 @@ jobs:
     <<: *test
   test-window-post:
     <<: *test
+  test-terminate:
+    <<: *test
   test-conformance:
     description: |
       Run tests using a corpus of interoperable test vectors for Filecoin 
@@ -476,9 +478,15 @@ workflows:
           test-suite-name: cli
           packages: "./cli/... ./cmd/... ./api/..."
       - test-window-post:
+          codecov-upload: true
           go-test-flags: "-run=TestWindowedPost"
           winpost-test: "1"
           test-suite-name: window-post
+      - test-terminate:
+          codecov-upload: true
+          go-test-flags: "-run=TestTerminate"
+          winpost-test: "1"
+          test-suite-name: terminate
       - test-short:
           go-test-flags: "--timeout 10m --short"
           test-suite-name: short

.codecov.yml

@@ -5,5 +5,15 @@ ignore:
   - "api/test/*"
   - "gen/**/*"
   - "gen/*"
+  - "cmd/lotus-shed/*"
+  - "cmd/tvx/*"
+  - "cmd/lotus-pcr/*"
+  - "cmd/tvx/*"
+  - "cmd/lotus-chainwatch/*"
+  - "cmd/lotus-health/*"
+  - "cmd/lotus-fountain/*"
+  - "cmd/lotus-townhall/*"
+  - "cmd/lotus-stats/*"
+  - "cmd/lotus-pcr/*"
 github_checks:
   annotations: false

CHANGELOG.md

@@ -1,5 +1,80 @@
 # Lotus changelog
 
+# 1.4.1 / 2021-01-20
+
+This is an optional Lotus release that introduces various improvements to the sealing, mining, and deal-making processes. In particular, [#5341](https://github.com/filecoin-project/lotus/pull/5341) introduces the ability for Lotus miners to terminate sectors.
+
+## Changes
+
+#### Core Lotus
+
+- fix(sync): enforce ForkLengthThreshold for synced chain (https://github.com/filecoin-project/lotus/pull/5182)
+- introduce memory watchdog; LOTUS_MAX_HEAP (https://github.com/filecoin-project/lotus/pull/5101)
+- Skip bootstrapping if no peers specified (https://github.com/filecoin-project/lotus/pull/5301)
+- Chainxchg write response timeout (https://github.com/filecoin-project/lotus/pull/5254)
+- update NewestNetworkVersion (https://github.com/filecoin-project/lotus/pull/5277)
+- fix(sync): remove checks bypass when we submit the block (https://github.com/filecoin-project/lotus/pull/4192)
+- chore: export vm.ShouldBurn (https://github.com/filecoin-project/lotus/pull/5355)
+- fix(sync): enforce fork len when changing head (https://github.com/filecoin-project/lotus/pull/5244)
+- Use 55th percentile instead of median for gas-price (https://github.com/filecoin-project/lotus/pull/5369)
+- update go-libp2p-pubsub to v0.4.1 (https://github.com/filecoin-project/lotus/pull/5329)
+
+#### Sealing
+
+- Sector termination support (https://github.com/filecoin-project/lotus/pull/5341)
+- update weight canSeal and canStore when attach (https://github.com/filecoin-project/lotus/pull/5242/files)
+- sector-storage/mock: improve mocked readpiece (https://github.com/filecoin-project/lotus/pull/5208)
+- Fix deadlock in runWorker in sched_worker.go (https://github.com/filecoin-project/lotus/pull/5251)
+- Skip checking terminated sectors provable (https://github.com/filecoin-project/lotus/pull/5217)
+- storagefsm: Fix unsealedInfoMap.lk init race (https://github.com/filecoin-project/lotus/pull/5319)
+- Multicore AddPiece CommP (https://github.com/filecoin-project/lotus/pull/5320)
+- storagefsm: Send correct event on ErrExpiredTicket in CommitFailed (https://github.com/filecoin-project/lotus/pull/5366)
+- expose StateSearchMessage on gateway (https://github.com/filecoin-project/lotus/pull/5382)
+- fix FileSize to return correct disk usage recursively (https://github.com/filecoin-project/lotus/pull/5384)
+
+#### Dealmaking
+
+- Better error message when withdrawing funds (https://github.com/filecoin-project/lotus/pull/5293)
+- add verbose for list transfers (https://github.com/filecoin-project/lotus/pull/5259)
+- cli - rename `client info` to `client balances` (https://github.com/filecoin-project/lotus/pull/5304)
+- Better CLI for wallet market withdraw and client info (https://github.com/filecoin-project/lotus/pull/5303)
+
+#### UX
+
+- correct flag usages for replace cmd (https://github.com/filecoin-project/lotus/pull/5255)
+- lotus state call will panic (https://github.com/filecoin-project/lotus/pull/5275)
+- fix get sector bug (https://github.com/filecoin-project/lotus/pull/4976)
+- feat: lotus wallet market add (adds funds to storage market actor) (https://github.com/filecoin-project/lotus/pull/5300)
+- Fix client flag parsing in client balances cli (https://github.com/filecoin-project/lotus/pull/5312)
+- delete slash-consensus miner (https://github.com/filecoin-project/lotus/pull/4577)
+- add fund sufficient check in send (https://github.com/filecoin-project/lotus/pull/5252)
+- enable parse and shorten negative FIL values (https://github.com/filecoin-project/lotus/pull/5315)
+- add limit and rate for chain noise (https://github.com/filecoin-project/lotus/pull/5223)
+- add bench env print (https://github.com/filecoin-project/lotus/pull/5222)
+- Implement full-node restore option (https://github.com/filecoin-project/lotus/pull/5362)
+- add color for token amount (https://github.com/filecoin-project/lotus/pull/5352)
+- correct log in maybeUseAddress (https://github.com/filecoin-project/lotus/pull/5359)
+- add slash-consensus from flag (https://github.com/filecoin-project/lotus/pull/5378)
+
+#### Testing
+
+- tvx extract: more tipset extraction goodness (https://github.com/filecoin-project/lotus/pull/5258)
+- Fix race in blockstore test suite (https://github.com/filecoin-project/lotus/pull/5297)
+
+
+#### Build & Networks
+
+- Remove LOTUS_DISABLE_V2_ACTOR_MIGRATION envvar (https://github.com/filecoin-project/lotus/pull/5289)
+- Create a calibnet build option (https://github.com/filecoin-project/lotus/pull/5288)
+- Calibnet: Set Orange epoch (https://github.com/filecoin-project/lotus/pull/5325)
+
+#### Management
+
+- Update SECURITY.md (https://github.com/filecoin-project/lotus/pull/5246)
+- README: Contribute section (https://github.com/filecoin-project/lotus/pull/5330)
+- README: refine Contribute section (https://github.com/filecoin-project/lotus/pull/5331)
+- Add misc tooling to codecov ignore list (https://github.com/filecoin-project/lotus/pull/5347)
+
 # 1.4.0 / 2020-12-19
 
 This is a MANDATORY hotfix release of Lotus that resolves a chain halt at height 336,459 caused by nondeterminism in specs-actors. The fix is to update actors to 2.3.3 in order to incorporate this fix https://github.com/filecoin-project/specs-actors/pull/1334.

Makefile

@@ -63,6 +63,9 @@ debug: lotus lotus-miner lotus-worker lotus-seed
 2k: GOFLAGS+=-tags=2k
 2k: lotus lotus-miner lotus-worker lotus-seed
 
+calibnet: GOFLAGS+=-tags=calibnet
+calibnet: lotus lotus-miner lotus-worker lotus-seed
+
 lotus: $(BUILD_DEPS)
 	rm -f lotus
 	go build $(GOFLAGS) -o lotus ./cmd/lotus

README.md

@@ -24,24 +24,31 @@ For instructions on how to build, install and setup lotus, please visit [https:/
 
 Please send an email to security@filecoin.org. See our [security policy](SECURITY.md) for more details.
 
-## Development
+## Related packages
 
-The main branches under development at the moment are:
-* [`master`](https://github.com/filecoin-project/lotus): current testnet.
-* [`next`](https://github.com/filecoin-project/lotus/tree/next): working branch with chain-breaking changes.
-* [`ntwk-calibration`](https://github.com/filecoin-project/lotus/tree/ntwk-calibration): devnet running one of `next` commits.
+These repos are independent and reusable modules, but are tightly integrated into Lotus to make up a fully featured Filecoin implementation:
 
-### Tracker
-
-All work is tracked via issues. An attempt at keeping an up-to-date view on remaining work towards Mainnet launch can be seen at the [lotus github project board](https://github.com/orgs/filecoin-project/projects/8). The issues labeled with `incentives` are there to identify the issues needed for Space Race launch.
-
-### Packages
-
-The lotus Filecoin implementation unfolds into the following packages:
-
-- [This repo](https://github.com/filecoin-project/lotus)
 - [go-fil-markets](https://github.com/filecoin-project/go-fil-markets) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/markets-shared-components-5daa144a7046a60001c6e253/board)
-- [spec-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
+- [specs-actors](https://github.com/filecoin-project/specs-actors) which has its own [kanban work tracker available here](https://app.zenhub.com/workspaces/actors-5ee6f3aa87591f0016c05685/board)
+
+## Contribute
+
+Lotus is a universally open project and welcomes contributions of all kinds: code, docs, and more. However, before making a contribution, we ask you to heed these recommendations:
+
+1. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs).
+2. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted.
+3. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn.
+
+When implementing a change:
+
+1. Adhere to the standard Go formatting guidelines, e.g. [Effective Go](https://golang.org/doc/effective_go.html). Run `go fmt`.
+2. Stick to the idioms and patterns used in the codebase. Familiar-looking code has a higher chance of being accepted than eerie code. Pay attention to commonly used variable and parameter names, avoidance of naked returns, error handling patterns, etc.
+3. Comments: follow the advice on the [Commentary](https://golang.org/doc/effective_go.html#commentary) section of Effective Go.
+4. Minimize code churn. Modify only what is strictly necessary. Well-encapsulated changesets will get a quicker response from maintainers.
+5. Lint your code with [`golangci-lint`](https://golangci-lint.run) (CI will reject your PR if unlinted).
+6. Add tests.
+7. Title the PR in a meaningful way and describe the rationale and the thought process in the PR description.
+8. Write clean, thoughtful, and detailed [commit messages](https://chris.beams.io/posts/git-commit/). This is even more important than the PR description, because commit messages are stored _inside_ the Git history. One good rule is: if you are happy posting the commit message as the PR description, then it's a good commit message.
 
 ## License
 

SECURITY.md

@@ -2,11 +2,11 @@
 
 ## Reporting a Vulnerability
 
-For *critical* bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md
+For reporting security vulnerabilities/bugs, please consult our Security Policy and Responsible Disclosure Program information at https://github.com/filecoin-project/community/blob/master/SECURITY.md. Security vulnerabilities should be reported via our [Vulnerability Reporting channels](https://github.com/filecoin-project/community/blob/master/SECURITY.md#vulnerability-reporting) and will be eligible for a [Bug Bounty](https://security.filecoin.io/bug-bounty/).
 
 Please try to provide a clear description of any bugs reported, along with how to reproduce the bug if possible. More detailed bug reports (especially those with a PoC included) will help us move forward much faster. Additionally, please avoid reporting bugs that already have open issues. Take a moment to search the issue list of the related GitHub repositories before writing up a new report.
 
-Here are some examples of bugs we would consider 'critical':
+Here are some examples of bugs we would consider to be security vulnerabilities:
 
 * If you can spend from a `multisig` wallet you do not control the keys for.
 * If you can cause a miner to be slashed without them actually misbehaving.
@@ -16,8 +16,8 @@ Here are some examples of bugs we would consider 'critical':
 * If you can craft a message that causes a persistent fork in the network.
 * If you can cause the total amount of Filecoin in the network to no longer be 2 billion.
 
-This is not an exhaustive list, but should provide some idea of what we consider 'critical'.
+This is not an exhaustive list, but should provide some idea of what we consider as a security vulnerability, . 
 
 ## Reporting a non security bug
 
-For non-critical bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). 
+For non-security bugs, please simply file a GitHub [issue](https://github.com/filecoin-project/lotus/issues/new?template=bug_report.md). 

api/api_full.go

@@ -391,6 +391,8 @@ type FullNode interface {
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error)
 	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
 	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
+	// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
+	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*MsgLookup, error)
 	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
 	// message arrives on chain, and gets to the indicated confidence depth.
 	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error)
@@ -515,6 +517,10 @@ type FullNode interface {
 	// along with the address removal.
 	MsigRemoveSigner(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error)
 
+	// MarketAddBalance adds funds to the market actor
+	MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error)
+	// MarketGetReserved gets the amount of funds that are currently reserved for the address
+	MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error)
 	// MarketReserveFunds reserves funds for a deal
 	MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error)
 	// MarketReleaseFunds releases funds reserved by MarketReserveFunds

api/api_gateway.go

@@ -39,6 +39,7 @@ type GatewayAPI interface {
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
+	StateSearchMsg(ctx context.Context, msg cid.Cid) (*MsgLookup, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
 	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error)

api/api_storage.go

@@ -65,7 +65,17 @@ type StorageMiner interface {
 	// SectorGetExpectedSealDuration gets the expected time for a sector to seal
 	SectorGetExpectedSealDuration(context.Context) (time.Duration, error)
 	SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error
+	// SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
+	// be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
 	SectorRemove(context.Context, abi.SectorNumber) error
+	// SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
+	// automatically removes it from storage
+	SectorTerminate(context.Context, abi.SectorNumber) error
+	// SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
+	// Returns null if message wasn't sent
+	SectorTerminateFlush(ctx context.Context) (*cid.Cid, error)
+	// SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
+	SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error)
 	SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error
 
 	StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error)
@@ -217,9 +227,12 @@ const (
 	PreCommitAddr AddrUse = iota
 	CommitAddr
 	PoStAddr
+
+	TerminateSectorsAddr
 )
 
 type AddressConfig struct {
 	PreCommitControl []address.Address
 	CommitControl    []address.Address
+	TerminateControl []address.Address
 }

api/apistruct/struct.go

@@ -206,6 +206,7 @@ type FullNodeStruct struct {
 		StateWaitMsg                       func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
 		StateWaitMsgLimited                func(context.Context, cid.Cid, uint64, abi.ChainEpoch) (*api.MsgLookup, error)                                      `perm:"read"`
 		StateSearchMsg                     func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
+		StateSearchMsgLimited              func(context.Context, cid.Cid, abi.ChainEpoch) (*api.MsgLookup, error)                                              `perm:"read"`
 		StateListMiners                    func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
 		StateListActors                    func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
 		StateMarketBalance                 func(context.Context, address.Address, types.TipSetKey) (api.MarketBalance, error)                                  `perm:"read"`
@@ -244,6 +245,8 @@ type FullNodeStruct struct {
 		MsigSwapCancel          func(context.Context, address.Address, address.Address, uint64, address.Address, address.Address) (cid.Cid, error)                               `perm:"sign"`
 		MsigRemoveSigner        func(ctx context.Context, msig address.Address, proposer address.Address, toRemove address.Address, decrease bool) (cid.Cid, error)              `perm:"sign"`
 
+		MarketAddBalance   func(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error)                 `perm:"sign"`
+		MarketGetReserved  func(ctx context.Context, addr address.Address) (types.BigInt, error)                                      `perm:"sign"`
 		MarketReserveFunds func(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) `perm:"sign"`
 		MarketReleaseFunds func(ctx context.Context, addr address.Address, amt types.BigInt) error                                    `perm:"sign"`
 		MarketWithdraw     func(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error)                 `perm:"sign"`
@@ -312,6 +315,9 @@ type StorageMinerStruct struct {
 		SectorGetExpectedSealDuration func(context.Context) (time.Duration, error)                                                  `perm:"read"`
 		SectorsUpdate                 func(context.Context, abi.SectorNumber, api.SectorState) error                                `perm:"admin"`
 		SectorRemove                  func(context.Context, abi.SectorNumber) error                                                 `perm:"admin"`
+		SectorTerminate               func(context.Context, abi.SectorNumber) error                                                 `perm:"admin"`
+		SectorTerminateFlush          func(ctx context.Context) (*cid.Cid, error)                                                   `perm:"admin"`
+		SectorTerminatePending        func(ctx context.Context) ([]abi.SectorID, error)                                             `perm:"admin"`
 		SectorMarkForUpgrade          func(ctx context.Context, id abi.SectorNumber) error                                          `perm:"admin"`
 
 		WorkerConnect func(context.Context, string) error                                `perm:"admin" retry:"true"` // TODO: worker perm
@@ -438,6 +444,7 @@ type GatewayStruct struct {
 		StateMinerProvingDeadline         func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 		StateMinerPower                   func(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
 		StateMarketBalance                func(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
+		StateSearchMsg                    func(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
 		StateMarketStorageDeal            func(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
 		StateNetworkVersion               func(ctx context.Context, tsk types.TipSetKey) (stnetwork.Version, error)
 		StateSectorGetInfo                func(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
@@ -1008,6 +1015,10 @@ func (c *FullNodeStruct) StateSearchMsg(ctx context.Context, msgc cid.Cid) (*api
 	return c.Internal.StateSearchMsg(ctx, msgc)
 }
 
+func (c *FullNodeStruct) StateSearchMsgLimited(ctx context.Context, msgc cid.Cid, limit abi.ChainEpoch) (*api.MsgLookup, error) {
+	return c.Internal.StateSearchMsgLimited(ctx, msgc, limit)
+}
+
 func (c *FullNodeStruct) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
 	return c.Internal.StateListMiners(ctx, tsk)
 }
@@ -1148,6 +1159,14 @@ func (c *FullNodeStruct) MsigRemoveSigner(ctx context.Context, msig address.Addr
 	return c.Internal.MsigRemoveSigner(ctx, msig, proposer, toRemove, decrease)
 }
 
+func (c *FullNodeStruct) MarketAddBalance(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) {
+	return c.Internal.MarketAddBalance(ctx, wallet, addr, amt)
+}
+
+func (c *FullNodeStruct) MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error) {
+	return c.Internal.MarketGetReserved(ctx, addr)
+}
+
 func (c *FullNodeStruct) MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) {
 	return c.Internal.MarketReserveFunds(ctx, wallet, addr, amt)
 }
@@ -1300,6 +1319,18 @@ func (c *StorageMinerStruct) SectorRemove(ctx context.Context, number abi.Sector
 	return c.Internal.SectorRemove(ctx, number)
 }
 
+func (c *StorageMinerStruct) SectorTerminate(ctx context.Context, number abi.SectorNumber) error {
+	return c.Internal.SectorTerminate(ctx, number)
+}
+
+func (c *StorageMinerStruct) SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) {
+	return c.Internal.SectorTerminateFlush(ctx)
+}
+
+func (c *StorageMinerStruct) SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) {
+	return c.Internal.SectorTerminatePending(ctx)
+}
+
 func (c *StorageMinerStruct) SectorMarkForUpgrade(ctx context.Context, number abi.SectorNumber) error {
 	return c.Internal.SectorMarkForUpgrade(ctx, number)
 }
@@ -1754,6 +1785,10 @@ func (g GatewayStruct) StateNetworkVersion(ctx context.Context, tsk types.TipSet
 	return g.Internal.StateNetworkVersion(ctx, tsk)
 }
 
+func (g GatewayStruct) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
+	return g.Internal.StateSearchMsg(ctx, msg)
+}
+
 func (g GatewayStruct) StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error) {
 	return g.Internal.StateSectorGetInfo(ctx, maddr, n, tsk)
 }

api/test/window_post.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/filecoin-project/lotus/extern/sector-storage/mock"
@@ -211,6 +212,7 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	}
 
 }
+
 func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration, nSectors int,
 	upgradeHeight abi.ChainEpoch) {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -428,3 +430,175 @@ func testWindowPostUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration,
 	sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz)
 	require.Equal(t, nSectors+GenesisPreseals-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged
 }
+
+func TestTerminate(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	nSectors := uint64(2)
+
+	n, sn := b(t, []FullNodeOpts{FullNodeWithActorsV2At(1)}, []StorageMiner{{Full: 0, Preseal: int(nSectors)}})
+
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	build.Clock.Sleep(time.Second)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for ctx.Err() == nil {
+			build.Clock.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+				if ctx.Err() != nil {
+					// context was canceled, ignore the error.
+					return
+				}
+				t.Error(err)
+			}
+		}
+	}()
+	defer func() {
+		cancel()
+		<-done
+	}()
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+
+	ssz, err := miner.ActorSectorSize(ctx, maddr)
+	require.NoError(t, err)
+
+	p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*nSectors))
+
+	fmt.Printf("Seal a sector\n")
+
+	pledgeSectors(t, ctx, miner, 1, 0, nil)
+
+	fmt.Printf("wait for power\n")
+
+	{
+		// Wait until proven.
+		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+		require.NoError(t, err)
+
+		waitUntil := di.PeriodStart + di.WPoStProvingPeriod + 2
+		fmt.Printf("End for head.Height > %d\n", waitUntil)
+
+		for {
+			head, err := client.ChainHead(ctx)
+			require.NoError(t, err)
+
+			if head.Height() > waitUntil {
+				fmt.Printf("Now head.Height = %d\n", head.Height())
+				break
+			}
+		}
+	}
+
+	nSectors++
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*nSectors))
+
+	fmt.Println("Terminate a sector")
+
+	toTerminate := abi.SectorNumber(3)
+
+	err = miner.SectorTerminate(ctx, toTerminate)
+	require.NoError(t, err)
+
+	msgTriggerred := false
+loop:
+	for {
+		si, err := miner.SectorsStatus(ctx, toTerminate, false)
+		require.NoError(t, err)
+
+		fmt.Println("state: ", si.State, msgTriggerred)
+
+		switch sealing.SectorState(si.State) {
+		case sealing.Terminating:
+			if !msgTriggerred {
+				{
+					p, err := miner.SectorTerminatePending(ctx)
+					require.NoError(t, err)
+					require.Len(t, p, 1)
+					require.Equal(t, abi.SectorNumber(3), p[0].Number)
+				}
+
+				c, err := miner.SectorTerminateFlush(ctx)
+				require.NoError(t, err)
+				if c != nil {
+					msgTriggerred = true
+					fmt.Println("terminate message:", c)
+
+					{
+						p, err := miner.SectorTerminatePending(ctx)
+						require.NoError(t, err)
+						require.Len(t, p, 0)
+					}
+				}
+			}
+		case sealing.TerminateWait, sealing.TerminateFinality, sealing.Removed:
+			break loop
+		}
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// check power decreased
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*(nSectors-1)))
+
+	// check in terminated set
+	{
+		parts, err := client.StateMinerPartitions(ctx, maddr, 1, types.EmptyTSK)
+		require.NoError(t, err)
+		require.Greater(t, len(parts), 0)
+
+		bflen := func(b bitfield.BitField) uint64 {
+			l, err := b.Count()
+			require.NoError(t, err)
+			return l
+		}
+
+		require.Equal(t, uint64(1), bflen(parts[0].AllSectors))
+		require.Equal(t, uint64(0), bflen(parts[0].LiveSectors))
+	}
+
+	di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+	for {
+		head, err := client.ChainHead(ctx)
+		require.NoError(t, err)
+
+		if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 {
+			fmt.Printf("Now head.Height = %d\n", head.Height())
+			break
+		}
+		build.Clock.Sleep(blocktime)
+	}
+	require.NoError(t, err)
+	fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
+
+	p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	require.Equal(t, p.MinerPower, p.TotalPower)
+	require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*(nSectors-1)))
+}

build/bootstrap.go

@@ -2,11 +2,9 @@ package build
 
 import (
 	"context"
-	"os"
 	"strings"
 
 	"github.com/filecoin-project/lotus/lib/addrutil"
-	"golang.org/x/xerrors"
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -17,24 +15,16 @@ func BuiltinBootstrap() ([]peer.AddrInfo, error) {
 		return nil, nil
 	}
 
-	var out []peer.AddrInfo
-
 	b := rice.MustFindBox("bootstrap")
-	err := b.Walk("", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return xerrors.Errorf("failed to walk box: %w", err)
+
+	if BootstrappersFile != "" {
+		spi := b.MustString(BootstrappersFile)
+		if spi == "" {
+			return nil, nil
 		}
 
-		if !strings.HasSuffix(path, ".pi") {
-			return nil
-		}
-		spi := b.MustString(path)
-		if spi == "" {
-			return nil
-		}
-		pi, err := addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n"))
-		out = append(out, pi...)
-		return err
-	})
-	return out, err
+		return addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n"))
+	}
+
+	return nil, nil
 }

build/bootstrap/calibnet.pi

@@ -0,0 +1,4 @@
+/dns4/bootstrap-0.calibration.fildev.network/tcp/1347/p2p/12D3KooWK1QYsm6iqyhgH7vqsbeoNoKHbT368h1JLHS1qYN36oyc
+/dns4/bootstrap-1.calibration.fildev.network/tcp/1347/p2p/12D3KooWKDyJZoPsNak1iYNN1GGmvGnvhyVbWBL6iusYfP3RpgYs
+/dns4/bootstrap-2.calibration.fildev.network/tcp/1347/p2p/12D3KooWJRSTnzABB6MYYEBbSTT52phQntVD1PpRTMh1xt9mh6yH
+/dns4/bootstrap-3.calibration.fildev.network/tcp/1347/p2p/12D3KooWQLi3kY6HnMYLUtwCe26zWMdNhniFgHVNn1DioQc7NiWv

build/genesis.go

@@ -14,7 +14,7 @@ func MaybeGenesis() []byte {
 		log.Warnf("loading built-in genesis: %s", err)
 		return nil
 	}
-	genBytes, err := builtinGen.Bytes("devnet.car")
+	genBytes, err := builtinGen.Bytes(GenesisFile)
 	if err != nil {
 		log.Warnf("loading built-in genesis: %s", err)
 	}

build/params_2k.go

@@ -3,13 +3,13 @@
 package build
 
 import (
-	"math"
-	"os"
-
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
+const BootstrappersFile = ""
+const GenesisFile = ""
+
 const UpgradeBreezeHeight = -1
 const BreezeGasTampingDuration = 0
 
@@ -18,8 +18,8 @@ const UpgradeIgnitionHeight = -2
 const UpgradeRefuelHeight = -3
 const UpgradeTapeHeight = -4
 
-var UpgradeActorsV2Height = abi.ChainEpoch(10)
-var UpgradeLiftoffHeight = abi.ChainEpoch(-5)
+const UpgradeActorsV2Height = 10
+const UpgradeLiftoffHeight = -5
 
 const UpgradeKumquatHeight = 15
 const UpgradeCalicoHeight = 20
@@ -36,11 +36,6 @@ func init() {
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
 	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
 
-	if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" {
-		UpgradeActorsV2Height = math.MaxInt64
-		UpgradeLiftoffHeight = 11
-	}
-
 	BuildType |= Build2k
 }
 

build/params_calibnet.go

@@ -0,0 +1,65 @@
+// +build calibnet
+
+package build
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+)
+
+var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
+	0: DrandMainnet,
+}
+
+const BootstrappersFile = "calibnet.pi"
+const GenesisFile = "calibnet.car"
+
+const UpgradeBreezeHeight = -1
+const BreezeGasTampingDuration = 120
+
+const UpgradeSmokeHeight = -2
+
+const UpgradeIgnitionHeight = -3
+const UpgradeRefuelHeight = -4
+
+var UpgradeActorsV2Height = abi.ChainEpoch(30)
+
+const UpgradeTapeHeight = 60
+
+// This signals our tentative epoch for mainnet launch. Can make it later, but not earlier.
+// Miners, clients, developers, custodians all need time to prepare.
+// We still have upgrades and state changes to do, but can happen after signaling timing here.
+const UpgradeLiftoffHeight = -5
+
+const UpgradeKumquatHeight = 90
+
+const UpgradeCalicoHeight = 92000
+const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 60)
+
+// 2020-12-17T19:00:00Z
+const UpgradeClausHeight = 161386
+
+// 2021-01-17T19:00:00Z
+const UpgradeOrangeHeight = 250666
+
+func init() {
+	policy.SetConsensusMinerMinPower(abi.NewStoragePower(10 << 30))
+	policy.SetSupportedProofTypes(
+		abi.RegisteredSealProof_StackedDrg512MiBV1,
+		abi.RegisteredSealProof_StackedDrg32GiBV1,
+		abi.RegisteredSealProof_StackedDrg64GiBV1,
+	)
+
+	SetAddressNetwork(address.Testnet)
+
+	Devnet = true
+}
+
+const BlockDelaySecs = uint64(builtin2.EpochDurationSeconds)
+
+const PropagationDelaySecs = uint64(6)
+
+// BootstrapPeerThreshold is the minimum number peers we need to track for a sync worker to start
+const BootstrapPeerThreshold = 4

build/params_mainnet.go

@@ -1,11 +1,11 @@
 // +build !debug
 // +build !2k
 // +build !testground
+// +build !calibnet
 
 package build
 
 import (
-	"math"
 	"os"
 
 	"github.com/filecoin-project/go-address"
@@ -19,7 +19,11 @@ var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	UpgradeSmokeHeight: DrandMainnet,
 }
 
+const BootstrappersFile = "mainnet.pi"
+const GenesisFile = "mainnet.car"
+
 const UpgradeBreezeHeight = 41280
+
 const BreezeGasTampingDuration = 120
 
 const UpgradeSmokeHeight = 51000
@@ -27,7 +31,7 @@ const UpgradeSmokeHeight = 51000
 const UpgradeIgnitionHeight = 94000
 const UpgradeRefuelHeight = 130800
 
-var UpgradeActorsV2Height = abi.ChainEpoch(138720)
+const UpgradeActorsV2Height = 138720
 
 const UpgradeTapeHeight = 140760
 
@@ -53,10 +57,6 @@ func init() {
 		SetAddressNetwork(address.Mainnet)
 	}
 
-	if os.Getenv("LOTUS_DISABLE_V2_ACTOR_MIGRATION") == "1" {
-		UpgradeActorsV2Height = math.MaxInt64
-	}
-
 	Devnet = false
 }
 

build/params_shared_vals.go

@@ -25,7 +25,7 @@ const UnixfsLinksPerLevel = 1024
 // Consensus / Network
 
 const AllowableClockDriftSecs = uint64(1)
-const NewestNetworkVersion = network.Version8
+const NewestNetworkVersion = network.Version9
 const ActorUpgradeNetworkVersion = network.Version4
 
 // Epochs

build/params_testground.go

@@ -97,11 +97,14 @@ var (
 		0: DrandMainnet,
 	}
 
-	NewestNetworkVersion       = network.Version8
+	NewestNetworkVersion       = network.Version9
 	ActorUpgradeNetworkVersion = network.Version4
 
 	Devnet      = true
 	ZeroAddress = MustParseAddress("f3yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaby2smx7a")
+
+	BootstrappersFile = ""
+	GenesisFile       = ""
 )
 
 const BootstrapPeerThreshold = 1

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.4.0"
+const BuildVersion = "1.4.1"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit
@@ -84,7 +84,7 @@ func VersionForType(nodeType NodeType) (Version, error) {
 // semver versions of the rpc api exposed
 var (
 	FullAPIVersion   = newVer(1, 0, 0)
-	MinerAPIVersion  = newVer(1, 0, 0)
+	MinerAPIVersion  = newVer(1, 0, 1)
 	WorkerAPIVersion = newVer(1, 0, 0)
 )
 

chain/actors/builtin/miner/miner.go

@@ -20,6 +20,7 @@ import (
 	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
 	miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
 )
 
 func init() {
@@ -42,6 +43,10 @@ var FaultDeclarationCutoff = miner0.FaultDeclarationCutoff
 
 const MinSectorExpiration = miner0.MinSectorExpiration
 
+// Not used / checked in v0
+var DeclarationsMax = miner2.DeclarationsMax
+var AddressedSectorsMax = miner2.AddressedSectorsMax
+
 func Load(store adt.Store, act *types.Actor) (st State, err error) {
 	switch act.Code {
 	case builtin0.StorageMinerActorCodeID:

chain/exchange/server.go

@@ -56,7 +56,11 @@ func (s *server) HandleStream(stream inet.Stream) {
 	}
 
 	_ = stream.SetDeadline(time.Now().Add(WriteResDeadline))
-	if err := cborutil.WriteCborRPC(stream, resp); err != nil {
+	buffered := bufio.NewWriter(stream)
+	if err = cborutil.WriteCborRPC(buffered, resp); err == nil {
+		err = buffered.Flush()
+	}
+	if err != nil {
 		_ = stream.SetDeadline(time.Time{})
 		log.Warnw("failed to write back response for handle stream",
 			"err", err, "peer", stream.Conn().RemotePeer())

chain/market/fundmanager.go

@@ -2,6 +2,7 @@ package market
 
 import (
 	"context"
+	"fmt"
 	"sync"
 
 	"github.com/filecoin-project/go-address"
@@ -129,6 +130,11 @@ func (fm *FundManager) Withdraw(ctx context.Context, wallet, addr address.Addres
 	return fm.getFundedAddress(addr).withdraw(ctx, wallet, amt)
 }
 
+// GetReserved returns the amount that is currently reserved for the address
+func (fm *FundManager) GetReserved(addr address.Address) abi.TokenAmount {
+	return fm.getFundedAddress(addr).getReserved()
+}
+
 // FundedAddressState keeps track of the state of an address with funds in the
 // datastore
 type FundedAddressState struct {
@@ -147,7 +153,7 @@ type fundedAddress struct {
 	env *fundManagerEnvironment
 	str *Store
 
-	lk    sync.Mutex
+	lk    sync.RWMutex
 	state *FundedAddressState
 
 	// Note: These request queues are ephemeral, they are not saved to store
@@ -183,6 +189,13 @@ func (a *fundedAddress) start() {
 	}
 }
 
+func (a *fundedAddress) getReserved() abi.TokenAmount {
+	a.lk.RLock()
+	defer a.lk.RUnlock()
+
+	return a.state.AmtReserved
+}
+
 func (a *fundedAddress) reserve(ctx context.Context, wallet address.Address, amt abi.TokenAmount) (cid.Cid, error) {
 	return a.requestAndWait(ctx, wallet, amt, &a.reservations)
 }
@@ -501,7 +514,13 @@ func (a *fundedAddress) processWithdrawals(withdrawals []*fundRequest) (msgCid c
 		// request with an error
 		newWithdrawalAmt := types.BigAdd(withdrawalAmt, amt)
 		if newWithdrawalAmt.GreaterThan(netAvail) {
-			err := xerrors.Errorf("insufficient funds for withdrawal of %d", amt)
+			msg := fmt.Sprintf("insufficient funds for withdrawal of %s: ", types.FIL(amt))
+			msg += fmt.Sprintf("net available (%s) = available (%s) - reserved (%s)",
+				types.FIL(types.BigSub(netAvail, withdrawalAmt)), types.FIL(avail), types.FIL(a.state.AmtReserved))
+			if !withdrawalAmt.IsZero() {
+				msg += fmt.Sprintf(" - queued withdrawals (%s)", types.FIL(withdrawalAmt))
+			}
+			err := xerrors.Errorf(msg)
 			a.debugf("%s", err)
 			req.Complete(cid.Undef, err)
 			continue

chain/stmgr/stmgr.go

@@ -639,7 +639,7 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid
 	}
 }
 
-func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) {
+func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid, lookbackLimit abi.ChainEpoch) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) {
 	msg, err := sm.cs.GetCMessage(mcid)
 	if err != nil {
 		return nil, nil, cid.Undef, fmt.Errorf("failed to load message: %w", err)
@@ -656,7 +656,7 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, mcid cid.Cid) (*ty
 		return head, r, foundMsg, nil
 	}
 
-	fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, LookbackNoLimit)
+	fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, lookbackLimit)
 
 	if err != nil {
 		log.Warnf("failed to look back through chain for message %s", mcid)

chain/store/store.go

@@ -363,7 +363,7 @@ func (cs *ChainStore) PutTipSet(ctx context.Context, ts *types.TipSet) error {
 
 // MaybeTakeHeavierTipSet evaluates the incoming tipset and locks it in our
 // internal state as our new head, if and only if it is heavier than the current
-// head.
+// head and does not exceed the maximum fork length.
 func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipSet) error {
 	cs.heaviestLk.Lock()
 	defer cs.heaviestLk.Unlock()
@@ -380,6 +380,15 @@ func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipS
 		// TODO: don't do this for initial sync. Now that we don't have a
 		// difference between 'bootstrap sync' and 'caught up' sync, we need
 		// some other heuristic.
+
+		exceeds, err := cs.exceedsForkLength(cs.heaviest, ts)
+		if err != nil {
+			return err
+		}
+		if exceeds {
+			return nil
+		}
+
 		return cs.takeHeaviestTipSet(ctx, ts)
 	} else if w.Equals(heaviestW) && !ts.Equals(cs.heaviest) {
 		log.Errorw("weight draw", "currTs", cs.heaviest, "ts", ts)
@@ -387,6 +396,67 @@ func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipS
 	return nil
 }
 
+// Check if the two tipsets have a fork length above `ForkLengthThreshold`.
+// `synced` is the head of the chain we are currently synced to and `external`
+// is the incoming tipset potentially belonging to a forked chain. It assumes
+// the external chain has already been validated and available in the ChainStore.
+// The "fast forward" case is covered in this logic as a valid fork of length 0.
+//
+// FIXME: We may want to replace some of the logic in `syncFork()` with this.
+//  `syncFork()` counts the length on both sides of the fork at the moment (we
+//  need to settle on that) but here we just enforce it on the `synced` side.
+func (cs *ChainStore) exceedsForkLength(synced, external *types.TipSet) (bool, error) {
+	if synced == nil || external == nil {
+		// FIXME: If `cs.heaviest` is nil we should just bypass the entire
+		//  `MaybeTakeHeavierTipSet` logic (instead of each of the called
+		//  functions having to handle the nil case on their own).
+		return false, nil
+	}
+
+	var err error
+	// `forkLength`: number of tipsets we need to walk back from the our `synced`
+	// chain to the common ancestor with the new `external` head in order to
+	// adopt the fork.
+	for forkLength := 0; forkLength < int(build.ForkLengthThreshold); forkLength++ {
+		// First walk back as many tipsets in the external chain to match the
+		// `synced` height to compare them. If we go past the `synced` height
+		// the subsequent match will fail but it will still be useful to get
+		// closer to the `synced` head parent's height in the next loop.
+		for external.Height() > synced.Height() {
+			if external.Height() == 0 {
+				// We reached the genesis of the external chain without a match;
+				// this is considered a fork outside the allowed limit (of "infinite"
+				// length).
+				return true, nil
+			}
+			external, err = cs.LoadTipSet(external.Parents())
+			if err != nil {
+				return false, xerrors.Errorf("failed to load parent tipset in external chain: %w", err)
+			}
+		}
+
+		// Now check if we arrived at the common ancestor.
+		if synced.Equals(external) {
+			return false, nil
+		}
+
+		// If we didn't, go back *one* tipset on the `synced` side (incrementing
+		// the `forkLength`).
+		if synced.Height() == 0 {
+			// Same check as the `external` side, if we reach the start (genesis)
+			// there is no common ancestor.
+			return true, nil
+		}
+		synced, err = cs.LoadTipSet(synced.Parents())
+		if err != nil {
+			return false, xerrors.Errorf("failed to load parent tipset in synced chain: %w", err)
+		}
+	}
+
+	// We traversed the fork length allowed without finding a common ancestor.
+	return true, nil
+}
+
 // ForceHeadSilent forces a chain head tipset without triggering a reorg
 // operation.
 //
@@ -524,9 +594,13 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet)
 // FlushValidationCache removes all results of block validation from the
 // chain metadata store. Usually the first step after a new chain import.
 func (cs *ChainStore) FlushValidationCache() error {
+	return FlushValidationCache(cs.ds)
+}
+
+func FlushValidationCache(ds datastore.Batching) error {
 	log.Infof("clearing block validation cache...")
 
-	dsWalk, err := cs.ds.Query(query.Query{
+	dsWalk, err := ds.Query(query.Query{
 		// Potential TODO: the validation cache is not a namespace on its own
 		// but is rather constructed as prefixed-key `foo:bar` via .Instance(), which
 		// in turn does not work with the filter, which can match only on `foo/bar`
@@ -546,7 +620,7 @@ func (cs *ChainStore) FlushValidationCache() error {
 		return xerrors.Errorf("failed to run key listing query: %w", err)
 	}
 
-	batch, err := cs.ds.Batch()
+	batch, err := ds.Batch()
 	if err != nil {
 		return xerrors.Errorf("failed to open a DS batch: %w", err)
 	}

chain/sync.go

@@ -250,18 +250,6 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) bool {
 
 	syncer.incoming.Pub(fts.TipSet().Blocks(), LocalIncoming)
 
-	if from == syncer.self {
-		// TODO: this is kindof a hack...
-		log.Debug("got block from ourselves")
-
-		if err := syncer.Sync(ctx, fts.TipSet()); err != nil {
-			log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err)
-			return false
-		}
-
-		return true
-	}
-
 	// TODO: IMPORTANT(GARBAGE) this needs to be put in the 'temporary' side of
 	// the blockstore
 	if err := syncer.store.PersistBlockHeaders(fts.TipSet().Blocks()...); err != nil {
@@ -1449,7 +1437,7 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know
 		return nil, ErrForkCheckpoint
 	}
 
-	// TODO: Does this mean we always ask for ForkLengthThreshold blocks from the network, even if we just need, like, 2?
+	// TODO: Does this mean we always ask for ForkLengthThreshold blocks from the network, even if we just need, like, 2? Yes.
 	// Would it not be better to ask in smaller chunks, given that an ~ForkLengthThreshold is very rare?
 	tips, err := syncer.Exchange.GetBlocks(ctx, incoming.Parents(), int(build.ForkLengthThreshold))
 	if err != nil {
@@ -1460,6 +1448,10 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know
 	if err != nil {
 		return nil, xerrors.Errorf("failed to load next local tipset: %w", err)
 	}
+	// Track the fork length on our side of the synced chain to enforce
+	// `ForkLengthThreshold`. Initialized to 1 because we already walked back
+	// one tipset from `known` (our synced head).
+	forkLengthInHead := 1
 
 	for cur := 0; cur < len(tips); {
 		if nts.Height() == 0 {
@@ -1476,6 +1468,13 @@ func (syncer *Syncer) syncFork(ctx context.Context, incoming *types.TipSet, know
 		if nts.Height() < tips[cur].Height() {
 			cur++
 		} else {
+			// Walk back one block in our synced chain to try to meet the fork's
+			// height.
+			forkLengthInHead++
+			if forkLengthInHead > int(build.ForkLengthThreshold) {
+				return nil, ErrForkTooLong
+			}
+
 			// We will be forking away from nts, check that it isn't checkpointed
 			if nts.Key() == chkpt {
 				return nil, ErrForkCheckpoint

chain/types/fil.go

@@ -26,7 +26,7 @@ func (f FIL) Unitless() string {
 var unitPrefixes = []string{"a", "f", "p", "n", "μ", "m"}
 
 func (f FIL) Short() string {
-	n := BigInt(f)
+	n := BigInt(f).Abs()
 
 	dn := uint64(1)
 	var prefix string
@@ -70,7 +70,7 @@ func (f FIL) UnmarshalText(text []byte) error {
 }
 
 func ParseFIL(s string) (FIL, error) {
-	suffix := strings.TrimLeft(s, ".1234567890")
+	suffix := strings.TrimLeft(s, "-.1234567890")
 	s = s[:len(s)-len(suffix)]
 	var attofil bool
 	if suffix != "" {

chain/types/fil_test.go

@@ -57,6 +57,52 @@ func TestFilShort(t *testing.T) {
 		{fil: "0.000221234", expect: "221.234 μFIL"},
 		{fil: "0.0002212344", expect: "221.234 μFIL"},
 		{fil: "0.00022123444", expect: "221.234 μFIL"},
+
+		{fil: "-1", expect: "-1 FIL"},
+		{fil: "-1.1", expect: "-1.1 FIL"},
+		{fil: "-12", expect: "-12 FIL"},
+		{fil: "-123", expect: "-123 FIL"},
+		{fil: "-123456", expect: "-123456 FIL"},
+		{fil: "-123.23", expect: "-123.23 FIL"},
+		{fil: "-123456.234", expect: "-123456.234 FIL"},
+		{fil: "-123456.2341234", expect: "-123456.234 FIL"},
+		{fil: "-123456.234123445", expect: "-123456.234 FIL"},
+
+		{fil: "-0.1", expect: "-100 mFIL"},
+		{fil: "-0.01", expect: "-10 mFIL"},
+		{fil: "-0.001", expect: "-1 mFIL"},
+
+		{fil: "-0.0001", expect: "-100 μFIL"},
+		{fil: "-0.00001", expect: "-10 μFIL"},
+		{fil: "-0.000001", expect: "-1 μFIL"},
+
+		{fil: "-0.0000001", expect: "-100 nFIL"},
+		{fil: "-0.00000001", expect: "-10 nFIL"},
+		{fil: "-0.000000001", expect: "-1 nFIL"},
+
+		{fil: "-0.0000000001", expect: "-100 pFIL"},
+		{fil: "-0.00000000001", expect: "-10 pFIL"},
+		{fil: "-0.000000000001", expect: "-1 pFIL"},
+
+		{fil: "-0.0000000000001", expect: "-100 fFIL"},
+		{fil: "-0.00000000000001", expect: "-10 fFIL"},
+		{fil: "-0.000000000000001", expect: "-1 fFIL"},
+
+		{fil: "-0.0000000000000001", expect: "-100 aFIL"},
+		{fil: "-0.00000000000000001", expect: "-10 aFIL"},
+		{fil: "-0.000000000000000001", expect: "-1 aFIL"},
+
+		{fil: "-0.0000012", expect: "-1.2 μFIL"},
+		{fil: "-0.00000123", expect: "-1.23 μFIL"},
+		{fil: "-0.000001234", expect: "-1.234 μFIL"},
+		{fil: "-0.0000012344", expect: "-1.234 μFIL"},
+		{fil: "-0.00000123444", expect: "-1.234 μFIL"},
+
+		{fil: "-0.0002212", expect: "-221.2 μFIL"},
+		{fil: "-0.00022123", expect: "-221.23 μFIL"},
+		{fil: "-0.000221234", expect: "-221.234 μFIL"},
+		{fil: "-0.0002212344", expect: "-221.234 μFIL"},
+		{fil: "-0.00022123444", expect: "-221.234 μFIL"},
 	} {
 		s := s
 		t.Run(s.fil, func(t *testing.T) {

chain/vm/vm.go

@@ -43,9 +43,11 @@ import (
 
 const MaxCallDepth = 4096
 
-var log = logging.Logger("vm")
-var actorLog = logging.Logger("actors")
-var gasOnActorExec = newGasCharge("OnActorExec", 0, 0)
+var (
+	log            = logging.Logger("vm")
+	actorLog       = logging.Logger("actors")
+	gasOnActorExec = newGasCharge("OnActorExec", 0, 0)
+)
 
 // stat counters
 var (
@@ -72,8 +74,10 @@ func ResolveToKeyAddr(state types.StateTree, cst cbor.IpldStore, addr address.Ad
 	return aast.PubkeyAddress()
 }
 
-var _ cbor.IpldBlockstore = (*gasChargingBlocks)(nil)
-var _ blockstore.Viewer = (*gasChargingBlocks)(nil)
+var (
+	_ cbor.IpldBlockstore = (*gasChargingBlocks)(nil)
+	_ blockstore.Viewer   = (*gasChargingBlocks)(nil)
+)
 
 type gasChargingBlocks struct {
 	chargeGas func(GasCharge)
@@ -194,9 +198,11 @@ func (vm *UnsafeVM) MakeRuntime(ctx context.Context, msg *types.Message) *Runtim
 	return vm.VM.makeRuntime(ctx, msg, nil)
 }
 
-type CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error)
-type NtwkVersionGetter func(context.Context, abi.ChainEpoch) network.Version
-type LookbackStateGetter func(context.Context, abi.ChainEpoch) (*state.StateTree, error)
+type (
+	CircSupplyCalculator func(context.Context, abi.ChainEpoch, *state.StateTree) (abi.TokenAmount, error)
+	NtwkVersionGetter    func(context.Context, abi.ChainEpoch) network.Version
+	LookbackStateGetter  func(context.Context, abi.ChainEpoch) (*state.StateTree, error)
+)
 
 type VM struct {
 	cstate         *state.StateTree
@@ -265,7 +271,6 @@ type ApplyRet struct {
 
 func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 	gasCharge *GasCharge, start time.Time) ([]byte, aerrors.ActorError, *Runtime) {
-
 	defer atomic.AddUint64(&StatSends, 1)
 
 	st := vm.cstate
@@ -563,7 +568,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet,
 		gasUsed = 0
 	}
 
-	burn, err := vm.shouldBurn(st, msg, errcode)
+	burn, err := vm.ShouldBurn(st, msg, errcode)
 	if err != nil {
 		return nil, xerrors.Errorf("deciding whether should burn failed: %w", err)
 	}
@@ -606,7 +611,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet,
 	}, nil
 }
 
-func (vm *VM) shouldBurn(st *state.StateTree, msg *types.Message, errcode exitcode.ExitCode) (bool, error) {
+func (vm *VM) ShouldBurn(st *state.StateTree, msg *types.Message, errcode exitcode.ExitCode) (bool, error) {
 	// Check to see if we should burn funds. We avoid burning on successful
 	// window post. This won't catch _indirect_ window post calls, but this
 	// is the best we can get for now.
@@ -737,7 +742,7 @@ func Copy(ctx context.Context, from, to blockstore.Blockstore, root cid.Cid) err
 		close(freeBufs)
 	}()
 
-	var batch = <-freeBufs
+	batch := <-freeBufs
 	batchCp := func(blk block.Block) error {
 		numBlocks++
 		totalCopySize += len(blk.RawData())

cli/chain.go

@@ -1106,8 +1106,8 @@ var slashConsensusFault = &cli.Command{
 	ArgsUsage: "[blockCid1 blockCid2]",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
-			Name:  "miner",
-			Usage: "Miner address",
+			Name:  "from",
+			Usage: "optionally specify the account to report consensus from",
 		},
 		&cli.StringFlag{
 			Name:  "extra",
@@ -1142,9 +1142,25 @@ var slashConsensusFault = &cli.Command{
 			return xerrors.Errorf("getting block 2: %w", err)
 		}
 
-		def, err := api.WalletDefaultAddress(ctx)
-		if err != nil {
-			return err
+		if b1.Miner != b2.Miner {
+			return xerrors.Errorf("block1.miner:%s block2.miner:%s", b1.Miner, b2.Miner)
+		}
+
+		var fromAddr address.Address
+		if from := cctx.String("from"); from == "" {
+			defaddr, err := api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return err
+			}
+
+			fromAddr = defaddr
+		} else {
+			addr, err := address.NewFromString(from)
+			if err != nil {
+				return err
+			}
+
+			fromAddr = addr
 		}
 
 		bh1, err := cborutil.Dump(b1)
@@ -1186,18 +1202,9 @@ var slashConsensusFault = &cli.Command{
 			return err
 		}
 
-		if cctx.String("miner") == "" {
-			return xerrors.Errorf("--miner flag is required")
-		}
-
-		maddr, err := address.NewFromString(cctx.String("miner"))
-		if err != nil {
-			return err
-		}
-
 		msg := &types.Message{
-			To:     maddr,
-			From:   def,
+			To:     b2.Miner,
+			From:   fromAddr,
 			Value:  types.NewInt(0),
 			Method: builtin.MethodsMiner.ReportConsensusFault,
 			Params: enc,

cli/client.go

@@ -91,7 +91,7 @@ var clientCmd = &cli.Command{
 		WithCategory("retrieval", clientRetrieveCmd),
 		WithCategory("util", clientCommPCmd),
 		WithCategory("util", clientCarGenCmd),
-		WithCategory("util", clientInfoCmd),
+		WithCategory("util", clientBalancesCmd),
 		WithCategory("util", clientListTransfers),
 		WithCategory("util", clientRestartTransfer),
 		WithCategory("util", clientCancelTransfer),
@@ -1732,9 +1732,9 @@ var clientGetDealCmd = &cli.Command{
 	},
 }
 
-var clientInfoCmd = &cli.Command{
-	Name:  "info",
-	Usage: "Print storage market client information",
+var clientBalancesCmd = &cli.Command{
+	Name:  "balances",
+	Usage: "Print storage market client balances",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "client",
@@ -1751,7 +1751,7 @@ var clientInfoCmd = &cli.Command{
 
 		var addr address.Address
 		if clientFlag := cctx.String("client"); clientFlag != "" {
-			ca, err := address.NewFromString("client")
+			ca, err := address.NewFromString(clientFlag)
 			if err != nil {
 				return err
 			}
@@ -1770,10 +1770,22 @@ var clientInfoCmd = &cli.Command{
 			return err
 		}
 
-		fmt.Printf("Client Market Info:\n")
+		reserved, err := api.MarketGetReserved(ctx, addr)
+		if err != nil {
+			return err
+		}
 
-		fmt.Printf("Locked Funds:\t%s\n", types.FIL(balance.Locked))
-		fmt.Printf("Escrowed Funds:\t%s\n", types.FIL(balance.Escrow))
+		avail := big.Sub(big.Sub(balance.Escrow, balance.Locked), reserved)
+		if avail.LessThan(big.Zero()) {
+			avail = big.Zero()
+		}
+
+		fmt.Printf("Client Market Balance for address %s:\n", addr)
+
+		fmt.Printf("  Escrowed Funds:        %s\n", types.FIL(balance.Escrow))
+		fmt.Printf("  Locked Funds:          %s\n", types.FIL(balance.Locked))
+		fmt.Printf("  Reserved Funds:        %s\n", types.FIL(reserved))
+		fmt.Printf("  Available to Withdraw: %s\n", types.FIL(avail))
 
 		return nil
 	},
@@ -1943,6 +1955,11 @@ var clientListTransfers = &cli.Command{
 	Name:  "list-transfers",
 	Usage: "List ongoing data transfers for deals",
 	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "verbose",
+			Aliases: []string{"v"},
+			Usage:   "print verbose transfer details",
+		},
 		&cli.BoolFlag{
 			Name:  "color",
 			Usage: "use color in display output",
@@ -1974,6 +1991,7 @@ var clientListTransfers = &cli.Command{
 			return err
 		}
 
+		verbose := cctx.Bool("verbose")
 		completed := cctx.Bool("completed")
 		color := cctx.Bool("color")
 		watch := cctx.Bool("watch")
@@ -1989,7 +2007,7 @@ var clientListTransfers = &cli.Command{
 
 				tm.MoveCursor(1, 1)
 
-				OutputDataTransferChannels(tm.Screen, channels, completed, color, showFailed)
+				OutputDataTransferChannels(tm.Screen, channels, verbose, completed, color, showFailed)
 
 				tm.Flush()
 
@@ -2014,13 +2032,13 @@ var clientListTransfers = &cli.Command{
 				}
 			}
 		}
-		OutputDataTransferChannels(os.Stdout, channels, completed, color, showFailed)
+		OutputDataTransferChannels(os.Stdout, channels, verbose, completed, color, showFailed)
 		return nil
 	},
 }
 
 // OutputDataTransferChannels generates table output for a list of channels
-func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChannel, completed bool, color bool, showFailed bool) {
+func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChannel, verbose, completed, color, showFailed bool) {
 	sort.Slice(channels, func(i, j int) bool {
 		return channels[i].TransferID < channels[j].TransferID
 	})
@@ -2050,7 +2068,7 @@ func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChann
 		tablewriter.Col("Voucher"),
 		tablewriter.NewLineCol("Message"))
 	for _, channel := range sendingChannels {
-		w.Write(toChannelOutput(color, "Sending To", channel))
+		w.Write(toChannelOutput(color, "Sending To", channel, verbose))
 	}
 	w.Flush(out) //nolint:errcheck
 
@@ -2064,7 +2082,7 @@ func OutputDataTransferChannels(out io.Writer, channels []lapi.DataTransferChann
 		tablewriter.Col("Voucher"),
 		tablewriter.NewLineCol("Message"))
 	for _, channel := range receivingChannels {
-		w.Write(toChannelOutput(color, "Receiving From", channel))
+		w.Write(toChannelOutput(color, "Receiving From", channel, verbose))
 	}
 	w.Flush(out) //nolint:errcheck
 }
@@ -2085,9 +2103,13 @@ func channelStatusString(useColor bool, status datatransfer.Status) string {
 	}
 }
 
-func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTransferChannel) map[string]interface{} {
-	rootCid := ellipsis(channel.BaseCID.String(), 8)
-	otherParty := ellipsis(channel.OtherPeer.String(), 8)
+func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTransferChannel, verbose bool) map[string]interface{} {
+	rootCid := channel.BaseCID.String()
+	otherParty := channel.OtherPeer.String()
+	if !verbose {
+		rootCid = ellipsis(rootCid, 8)
+		otherParty = ellipsis(otherParty, 8)
+	}
 
 	initiated := "N"
 	if channel.IsInitiator {
@@ -2095,7 +2117,7 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr
 	}
 
 	voucher := channel.Voucher
-	if len(voucher) > 40 {
+	if len(voucher) > 40 && !verbose {
 		voucher = ellipsis(voucher, 37)
 	}
 

cli/mpool.go

@@ -362,15 +362,15 @@ var mpoolReplaceCmd = &cli.Command{
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "gas-feecap",
-			Usage: "gas feecap for new message",
+			Usage: "gas feecap for new message (burn and pay to miner, attoFIL/GasUnit)",
 		},
 		&cli.StringFlag{
 			Name:  "gas-premium",
-			Usage: "gas price for new message",
+			Usage: "gas price for new message (pay to miner, attoFIL/GasUnit)",
 		},
 		&cli.Int64Flag{
 			Name:  "gas-limit",
-			Usage: "gas price for new message",
+			Usage: "gas limit for new message (GasUnit)",
 		},
 		&cli.BoolFlag{
 			Name:  "auto",
@@ -378,7 +378,7 @@ var mpoolReplaceCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "max-fee",
-			Usage: "Spend up to X FIL for this message (applicable for auto mode)",
+			Usage: "Spend up to X attoFIL for this message (applicable for auto mode)",
 		},
 	},
 	ArgsUsage: "<from nonce> | <message-cid>",

cli/send.go

@@ -15,6 +15,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -51,7 +52,7 @@ var sendCmd = &cli.Command{
 		&cli.Uint64Flag{
 			Name:  "method",
 			Usage: "specify method to invoke",
-			Value: 0,
+			Value: uint64(builtin.MethodSend),
 		},
 		&cli.StringFlag{
 			Name:  "params-json",
@@ -61,6 +62,10 @@ var sendCmd = &cli.Command{
 			Name:  "params-hex",
 			Usage: "specify invocation parameters in hex",
 		},
+		&cli.BoolFlag{
+			Name:  "force",
+			Usage: "must be specified for the action to take effect if maybe SysErrInsufficientFunds etc",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 2 {
@@ -143,6 +148,20 @@ var sendCmd = &cli.Command{
 			Params:     params,
 		}
 
+		if !cctx.Bool("force") {
+			// Funds insufficient check
+			fromBalance, err := api.WalletBalance(ctx, msg.From)
+			if err != nil {
+				return err
+			}
+			totalCost := types.BigAdd(types.BigMul(msg.GasFeeCap, types.NewInt(uint64(msg.GasLimit))), msg.Value)
+
+			if fromBalance.LessThan(totalCost) {
+				fmt.Printf("WARNING: From balance %s less than total cost %s\n", types.FIL(fromBalance), types.FIL(totalCost))
+				return fmt.Errorf("--force must be specified for this action to have an effect; you have been warned")
+			}
+		}
+
 		if cctx.IsSet("nonce") {
 			msg.Nonce = cctx.Uint64("nonce")
 			sm, err := api.WalletSignMessage(ctx, fromAddr, msg)

cli/state.go

@@ -1617,7 +1617,7 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er
 		return nil, fmt.Errorf("unknown method %d for actor %s", method, act)
 	}
 
-	paramObj := methodMeta.Params
+	paramObj := methodMeta.Params.Elem()
 	if paramObj.NumField() != len(args) {
 		return nil, fmt.Errorf("not enough arguments given to call that method (expecting %d)", paramObj.NumField())
 	}

cli/wallet.go

@@ -509,6 +509,7 @@ var walletMarket = &cli.Command{
 	Usage: "Interact with market balances",
 	Subcommands: []*cli.Command{
 		walletMarketWithdraw,
+		walletMarketAdd,
 	},
 }
 
@@ -518,13 +519,13 @@ var walletMarketWithdraw = &cli.Command{
 	ArgsUsage: "[amount (FIL) optional, otherwise will withdraw max available]",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
-			Name:    "from",
-			Usage:   "Specify address to withdraw funds from, otherwise it will use the default wallet address",
-			Aliases: []string{"f"},
+			Name:    "wallet",
+			Usage:   "Specify address to withdraw funds to, otherwise it will use the default wallet address",
+			Aliases: []string{"w"},
 		},
 		&cli.StringFlag{
 			Name:    "address",
-			Usage:   "Market address to withdraw from (account or miner actor address, defaults to --from address)",
+			Usage:   "Market address to withdraw from (account or miner actor address, defaults to --wallet address)",
 			Aliases: []string{"a"},
 		},
 	},
@@ -536,6 +537,123 @@ var walletMarketWithdraw = &cli.Command{
 		defer closer()
 		ctx := ReqContext(cctx)
 
+		var wallet address.Address
+		if cctx.String("wallet") != "" {
+			wallet, err = address.NewFromString(cctx.String("wallet"))
+			if err != nil {
+				return xerrors.Errorf("parsing from address: %w", err)
+			}
+		} else {
+			wallet, err = api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return xerrors.Errorf("getting default wallet address: %w", err)
+			}
+		}
+
+		addr := wallet
+		if cctx.String("address") != "" {
+			addr, err = address.NewFromString(cctx.String("address"))
+			if err != nil {
+				return xerrors.Errorf("parsing market address: %w", err)
+			}
+		}
+
+		// Work out if there are enough unreserved, unlocked funds to withdraw
+		bal, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("getting market balance for address %s: %w", addr.String(), err)
+		}
+
+		reserved, err := api.MarketGetReserved(ctx, addr)
+		if err != nil {
+			return xerrors.Errorf("getting market reserved amount for address %s: %w", addr.String(), err)
+		}
+
+		avail := big.Subtract(big.Subtract(bal.Escrow, bal.Locked), reserved)
+
+		notEnoughErr := func(msg string) error {
+			return xerrors.Errorf("%s; "+
+				"available (%s) = escrow (%s) - locked (%s) - reserved (%s)",
+				msg, types.FIL(avail), types.FIL(bal.Escrow), types.FIL(bal.Locked), types.FIL(reserved))
+		}
+
+		if avail.IsZero() || avail.LessThan(big.Zero()) {
+			avail = big.Zero()
+			return notEnoughErr("no funds available to withdraw")
+		}
+
+		// Default to withdrawing all available funds
+		amt := avail
+
+		// If there was an amount argument, only withdraw that amount
+		if cctx.Args().Present() {
+			f, err := types.ParseFIL(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("parsing 'amount' argument: %w", err)
+			}
+
+			amt = abi.TokenAmount(f)
+		}
+
+		// Check the amount is positive
+		if amt.IsZero() || amt.LessThan(big.Zero()) {
+			return xerrors.Errorf("amount must be > 0")
+		}
+
+		// Check there are enough available funds
+		if amt.GreaterThan(avail) {
+			msg := fmt.Sprintf("can't withdraw more funds than available; requested: %s", types.FIL(amt))
+			return notEnoughErr(msg)
+		}
+
+		fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), wallet.String())
+		smsg, err := api.MarketWithdraw(ctx, wallet, addr, amt)
+		if err != nil {
+			return xerrors.Errorf("fund manager withdraw error: %w", err)
+		}
+
+		fmt.Printf("WithdrawBalance message cid: %s\n", smsg)
+
+		return nil
+	},
+}
+
+var walletMarketAdd = &cli.Command{
+	Name:      "add",
+	Usage:     "Add funds to the Storage Market Actor",
+	ArgsUsage: "<amount>",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "from",
+			Usage:   "Specify address to move funds from, otherwise it will use the default wallet address",
+			Aliases: []string{"f"},
+		},
+		&cli.StringFlag{
+			Name:    "address",
+			Usage:   "Market address to move funds to (account or miner actor address, defaults to --from address)",
+			Aliases: []string{"a"},
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("getting node API: %w", err)
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		// Get amount param
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass amount to add")
+		}
+		f, err := types.ParseFIL(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("parsing 'amount' argument: %w", err)
+		}
+
+		amt := abi.TokenAmount(f)
+
+		// Get from param
 		var from address.Address
 		if cctx.String("from") != "" {
 			from, err = address.NewFromString(cctx.String("from"))
@@ -549,6 +667,7 @@ var walletMarketWithdraw = &cli.Command{
 			}
 		}
 
+		// Get address param
 		addr := from
 		if cctx.String("address") != "" {
 			addr, err = address.NewFromString(cctx.String("address"))
@@ -557,38 +676,14 @@ var walletMarketWithdraw = &cli.Command{
 			}
 		}
 
-		bal, err := api.StateMarketBalance(ctx, addr, types.EmptyTSK)
+		// Add balance to market actor
+		fmt.Printf("Submitting Add Balance message for amount %s for address %s\n", types.FIL(amt), addr)
+		smsg, err := api.MarketAddBalance(ctx, from, addr, amt)
 		if err != nil {
-			return xerrors.Errorf("getting market balance for address %s: %w", addr.String(), err)
+			return xerrors.Errorf("add balance error: %w", err)
 		}
 
-		avail := big.Subtract(bal.Escrow, bal.Locked)
-		amt := avail
-
-		if cctx.Args().Present() {
-			f, err := types.ParseFIL(cctx.Args().First())
-			if err != nil {
-				return xerrors.Errorf("parsing 'amount' argument: %w", err)
-			}
-
-			amt = abi.TokenAmount(f)
-		}
-
-		if amt.GreaterThan(avail) {
-			return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amt), types.FIL(avail))
-		}
-
-		if avail.IsZero() {
-			return xerrors.Errorf("zero unlocked funds available to withdraw")
-		}
-
-		fmt.Printf("Submitting WithdrawBalance message for amount %s for address %s\n", types.FIL(amt), from.String())
-		smsg, err := api.MarketWithdraw(ctx, from, addr, amt)
-		if err != nil {
-			return xerrors.Errorf("fund manager withdraw error: %w", err)
-		}
-
-		fmt.Printf("WithdrawBalance message cid: %s\n", smsg)
+		fmt.Printf("AddBalance message cid: %s\n", smsg)
 
 		return nil
 	},

cmd/chain-noise/main.go

@@ -27,6 +27,16 @@ func main() {
 				Hidden:  true,
 				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
 			},
+			&cli.IntFlag{
+				Name:  "limit",
+				Usage: "spam transaction count limit, <= 0 is no limit",
+				Value: 0,
+			},
+			&cli.IntFlag{
+				Name:  "rate",
+				Usage: "spam transaction rate, count per second",
+				Value: 5,
+			},
 		},
 		Commands: []*cli.Command{runCmd},
 	}
@@ -52,11 +62,17 @@ var runCmd = &cli.Command{
 		defer closer()
 		ctx := lcli.ReqContext(cctx)
 
-		return sendSmallFundsTxs(ctx, api, addr, 5)
+		rate := cctx.Int("rate")
+		if rate <= 0 {
+			rate = 5
+		}
+		limit := cctx.Int("limit")
+
+		return sendSmallFundsTxs(ctx, api, addr, rate, limit)
 	},
 }
 
-func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Address, rate int) error {
+func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Address, rate, limit int) error {
 	var sendSet []address.Address
 	for i := 0; i < 20; i++ {
 		naddr, err := api.WalletNew(ctx, types.KTSecp256k1)
@@ -66,9 +82,14 @@ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Addre
 
 		sendSet = append(sendSet, naddr)
 	}
+	count := limit
 
 	tick := build.Clock.Ticker(time.Second / time.Duration(rate))
 	for {
+		if count <= 0 && limit > 0 {
+			fmt.Printf("%d messages sent.\n", limit)
+			return nil
+		}
 		select {
 		case <-tick.C:
 			msg := &types.Message{
@@ -81,6 +102,7 @@ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Addre
 			if err != nil {
 				return err
 			}
+			count--
 			fmt.Println("Message sent: ", smsg.Cid())
 		case <-ctx.Done():
 			return nil

cmd/lotus-bench/main.go

@@ -40,6 +40,8 @@ import (
 var log = logging.Logger("lotus-bench")
 
 type BenchResults struct {
+	EnvVar map[string]string
+
 	SectorSize   abi.SectorSize
 	SectorNumber int
 
@@ -446,6 +448,15 @@ var sealBenchCmd = &cli.Command{
 			bo.VerifyWindowPostHot = verifyWindowpost2.Sub(verifyWindowpost1)
 		}
 
+		bo.EnvVar = make(map[string]string)
+		for _, envKey := range []string{"BELLMAN_NO_GPU", "FIL_PROOFS_MAXIMIZE_CACHING", "FIL_PROOFS_USE_GPU_COLUMN_BUILDER",
+			"FIL_PROOFS_USE_GPU_TREE_BUILDER", "FIL_PROOFS_USE_MULTICORE_SDR", "BELLMAN_CUSTOM_GPU"} {
+			envValue, found := os.LookupEnv(envKey)
+			if found {
+				bo.EnvVar[envKey] = envValue
+			}
+		}
+
 		if c.Bool("json-out") {
 			data, err := json.MarshalIndent(bo, "", "  ")
 			if err != nil {
@@ -454,6 +465,10 @@ var sealBenchCmd = &cli.Command{
 
 			fmt.Println(string(data))
 		} else {
+			fmt.Println("environment variable list:")
+			for envKey, envValue := range bo.EnvVar {
+				fmt.Printf("%s=%s\n", envKey, envValue)
+			}
 			fmt.Printf("----\nresults (v28) SectorSize:(%d), SectorNumber:(%d)\n", sectorSize, sectorNumber)
 			if robench == "" {
 				fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingSum.AddPiece, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.AddPiece))

cmd/lotus-gateway/api.go

@@ -57,6 +57,7 @@ type gatewayDepsAPI interface {
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
 	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
+	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error)
 	StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error)
 	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
@@ -299,6 +300,10 @@ func (a *GatewayAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKe
 	return a.api.StateNetworkVersion(ctx, tsk)
 }
 
+func (a *GatewayAPI) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
+	return a.api.StateSearchMsgLimited(ctx, msg, a.stateWaitLookbackLimit)
+}
+
 func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) {
 	return a.api.StateWaitMsgLimited(ctx, msg, confidence, a.stateWaitLookbackLimit)
 }

cmd/lotus-storage-miner/info.go

@@ -222,7 +222,7 @@ func infoCmdAct(cctx *cli.Context) error {
 	fmt.Printf("      PreCommit:  %s\n", types.FIL(lockedFunds.PreCommitDeposits).Short())
 	fmt.Printf("      Pledge:     %s\n", types.FIL(lockedFunds.InitialPledgeRequirement).Short())
 	fmt.Printf("      Vesting:    %s\n", types.FIL(lockedFunds.VestingFunds).Short())
-	color.Green("      Available:  %s", types.FIL(availBalance).Short())
+	colorTokenAmount("      Available:  %s\n", availBalance)
 
 	mb, err := api.StateMarketBalance(ctx, maddr, types.EmptyTSK)
 	if err != nil {
@@ -232,7 +232,7 @@ func infoCmdAct(cctx *cli.Context) error {
 
 	fmt.Printf("Market Balance:   %s\n", types.FIL(mb.Escrow).Short())
 	fmt.Printf("       Locked:    %s\n", types.FIL(mb.Locked).Short())
-	color.Green("       Available: %s\n", types.FIL(big.Sub(mb.Escrow, mb.Locked)).Short())
+	colorTokenAmount("       Available: %s\n", big.Sub(mb.Escrow, mb.Locked))
 
 	wb, err := api.WalletBalance(ctx, mi.Worker)
 	if err != nil {
@@ -253,7 +253,7 @@ func infoCmdAct(cctx *cli.Context) error {
 
 		fmt.Printf("       Control:   %s\n", types.FIL(cbsum).Short())
 	}
-	fmt.Printf("Total Spendable:  %s\n", color.YellowString(types.FIL(spendable).Short()))
+	colorTokenAmount("Total Spendable:  %s\n", spendable)
 
 	fmt.Println()
 
@@ -298,6 +298,10 @@ var stateList = []stateMeta{
 	{col: color.FgYellow, state: sealing.CommitWait},
 	{col: color.FgYellow, state: sealing.FinalizeSector},
 
+	{col: color.FgCyan, state: sealing.Terminating},
+	{col: color.FgCyan, state: sealing.TerminateWait},
+	{col: color.FgCyan, state: sealing.TerminateFinality},
+	{col: color.FgCyan, state: sealing.TerminateFailed},
 	{col: color.FgCyan, state: sealing.Removing},
 	{col: color.FgCyan, state: sealing.Removed},
 
@@ -355,3 +359,13 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) error {
 
 	return nil
 }
+
+func colorTokenAmount(format string, amount abi.TokenAmount) {
+	if amount.GreaterThan(big.Zero()) {
+		color.Green(format, types.FIL(amount).Short())
+	} else if amount.Equals(big.Zero()) {
+		color.Yellow(format, types.FIL(amount).Short())
+	} else {
+		color.Red(format, types.FIL(amount).Short())
+	}
+}

cmd/lotus-storage-miner/market.go

@@ -744,6 +744,11 @@ var transfersListCmd = &cli.Command{
 	Name:  "list",
 	Usage: "List ongoing data transfers for this miner",
 	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "verbose",
+			Aliases: []string{"v"},
+			Usage:   "print verbose transfer details",
+		},
 		&cli.BoolFlag{
 			Name:  "color",
 			Usage: "use color in display output",
@@ -775,6 +780,7 @@ var transfersListCmd = &cli.Command{
 			return err
 		}
 
+		verbose := cctx.Bool("verbose")
 		completed := cctx.Bool("completed")
 		color := cctx.Bool("color")
 		watch := cctx.Bool("watch")
@@ -790,7 +796,7 @@ var transfersListCmd = &cli.Command{
 
 				tm.MoveCursor(1, 1)
 
-				lcli.OutputDataTransferChannels(tm.Screen, channels, completed, color, showFailed)
+				lcli.OutputDataTransferChannels(tm.Screen, channels, verbose, completed, color, showFailed)
 
 				tm.Flush()
 
@@ -815,7 +821,7 @@ var transfersListCmd = &cli.Command{
 				}
 			}
 		}
-		lcli.OutputDataTransferChannels(os.Stdout, channels, completed, color, showFailed)
+		lcli.OutputDataTransferChannels(os.Stdout, channels, verbose, completed, color, showFailed)
 		return nil
 	},
 }

cmd/lotus-storage-miner/proving.go

@@ -446,7 +446,7 @@ var provingCheckProvableCmd = &cli.Command{
 		for parIdx, par := range partitions {
 			sectors := make(map[abi.SectorNumber]struct{})
 
-			sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.AllSectors, types.EmptyTSK)
+			sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.LiveSectors, types.EmptyTSK)
 			if err != nil {
 				return err
 			}

cmd/lotus-storage-miner/sectors.go

@@ -35,6 +35,7 @@ var sectorsCmd = &cli.Command{
 		sectorsRefsCmd,
 		sectorsUpdateCmd,
 		sectorsPledgeCmd,
+		sectorsTerminateCmd,
 		sectorsRemoveCmd,
 		sectorsMarkForUpgradeCmd,
 		sectorsStartSealCmd,
@@ -396,9 +397,123 @@ var sectorsRefsCmd = &cli.Command{
 	},
 }
 
+var sectorsTerminateCmd = &cli.Command{
+	Name:      "terminate",
+	Usage:     "Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector)",
+	ArgsUsage: "<sectorNum>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "pass this flag if you know what you are doing",
+		},
+	},
+	Subcommands: []*cli.Command{
+		sectorsTerminateFlushCmd,
+		sectorsTerminatePendingCmd,
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("really-do-it") {
+			return xerrors.Errorf("pass --really-do-it to confirm this action")
+		}
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+		if cctx.Args().Len() != 1 {
+			return xerrors.Errorf("must pass sector number")
+		}
+
+		id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("could not parse sector number: %w", err)
+		}
+
+		return nodeApi.SectorTerminate(ctx, abi.SectorNumber(id))
+	},
+}
+
+var sectorsTerminateFlushCmd = &cli.Command{
+	Name:  "flush",
+	Usage: "Send a terminate message if there are sectors queued for termination",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		mcid, err := nodeApi.SectorTerminateFlush(ctx)
+		if err != nil {
+			return err
+		}
+
+		if mcid == nil {
+			return xerrors.New("no sectors were queued for termination")
+		}
+
+		fmt.Println(mcid)
+
+		return nil
+	},
+}
+
+var sectorsTerminatePendingCmd = &cli.Command{
+	Name:  "pending",
+	Usage: "List sector numbers of sectors pending termination",
+	Action: func(cctx *cli.Context) error {
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		api, nCloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer nCloser()
+		ctx := lcli.ReqContext(cctx)
+
+		pending, err := nodeApi.SectorTerminatePending(ctx)
+		if err != nil {
+			return err
+		}
+
+		maddr, err := nodeApi.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		dl, err := api.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("getting proving deadline info failed: %w", err)
+		}
+
+		for _, id := range pending {
+			loc, err := api.StateSectorPartition(ctx, maddr, id.Number, types.EmptyTSK)
+			if err != nil {
+				return xerrors.Errorf("finding sector partition: %w", err)
+			}
+
+			fmt.Print(id.Number)
+
+			if loc.Deadline == (dl.Index+1)%miner.WPoStPeriodDeadlines || // not in next (in case the terminate message takes a while to get on chain)
+				loc.Deadline == dl.Index || // not in current
+				(loc.Deadline+1)%miner.WPoStPeriodDeadlines == dl.Index { // not in previous
+				fmt.Print(" (in proving window)")
+			}
+			fmt.Println()
+		}
+
+		return nil
+	},
+}
+
 var sectorsRemoveCmd = &cli.Command{
 	Name:      "remove",
-	Usage:     "Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector)",
+	Usage:     "Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty))",
 	ArgsUsage: "<sectorNum>",
 	Flags: []cli.Flag{
 		&cli.BoolFlag{

cmd/lotus/backup.go

@@ -1,14 +1,121 @@
 package main
 
 import (
+	"os"
+
+	dstore "github.com/ipfs/go-datastore"
+	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+	"gopkg.in/cheggaaa/pb.v1"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
+	"github.com/filecoin-project/lotus/chain/store"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/backupds"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
 var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) {
 	return lcli.GetFullNodeAPI(cctx)
 })
+
+func restore(cctx *cli.Context, r repo.Repo) error {
+	bf, err := homedir.Expand(cctx.Path("restore"))
+	if err != nil {
+		return xerrors.Errorf("expand backup file path: %w", err)
+	}
+
+	st, err := os.Stat(bf)
+	if err != nil {
+		return xerrors.Errorf("stat backup file (%s): %w", bf, err)
+	}
+
+	f, err := os.Open(bf)
+	if err != nil {
+		return xerrors.Errorf("opening backup file: %w", err)
+	}
+	defer f.Close() // nolint:errcheck
+
+	lr, err := r.Lock(repo.FullNode)
+	if err != nil {
+		return err
+	}
+	defer lr.Close() // nolint:errcheck
+
+	if cctx.IsSet("restore-config") {
+		log.Info("Restoring config")
+
+		cf, err := homedir.Expand(cctx.String("restore-config"))
+		if err != nil {
+			return xerrors.Errorf("expanding config path: %w", err)
+		}
+
+		_, err = os.Stat(cf)
+		if err != nil {
+			return xerrors.Errorf("stat config file (%s): %w", cf, err)
+		}
+
+		var cerr error
+		err = lr.SetConfig(func(raw interface{}) {
+			rcfg, ok := raw.(*config.FullNode)
+			if !ok {
+				cerr = xerrors.New("expected miner config")
+				return
+			}
+
+			ff, err := config.FromFile(cf, rcfg)
+			if err != nil {
+				cerr = xerrors.Errorf("loading config: %w", err)
+				return
+			}
+
+			*rcfg = *ff.(*config.FullNode)
+		})
+		if cerr != nil {
+			return cerr
+		}
+		if err != nil {
+			return xerrors.Errorf("setting config: %w", err)
+		}
+
+	} else {
+		log.Warn("--restore-config NOT SET, WILL USE DEFAULT VALUES")
+	}
+
+	log.Info("Restoring metadata backup")
+
+	mds, err := lr.Datastore("/metadata")
+	if err != nil {
+		return err
+	}
+
+	bar := pb.New64(st.Size())
+	br := bar.NewProxyReader(f)
+	bar.ShowTimeLeft = true
+	bar.ShowPercent = true
+	bar.ShowSpeed = true
+	bar.Units = pb.U_BYTES
+
+	bar.Start()
+	err = backupds.RestoreInto(br, mds)
+	bar.Finish()
+
+	if err != nil {
+		return xerrors.Errorf("restoring metadata: %w", err)
+	}
+
+	log.Info("Resetting chainstore metadata")
+
+	chainHead := dstore.NewKey("head")
+	if err := mds.Delete(chainHead); err != nil {
+		return xerrors.Errorf("clearing chain head: %w", err)
+	}
+	if err := store.FlushValidationCache(mds); err != nil {
+		return xerrors.Errorf("clearing chain validation cache: %w", err)
+	}
+
+	return nil
+}

cmd/lotus/daemon.go

@@ -144,6 +144,14 @@ var DaemonCmd = &cli.Command{
 			Name:  "api-max-req-size",
 			Usage: "maximum API request size accepted by the JSON RPC server",
 		},
+		&cli.PathFlag{
+			Name:  "restore",
+			Usage: "restore from backup file",
+		},
+		&cli.PathFlag{
+			Name:  "restore-config",
+			Usage: "config file to use when restoring from backup",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		isLite := cctx.Bool("lite")
@@ -203,9 +211,11 @@ var DaemonCmd = &cli.Command{
 			r.SetConfigPath(cctx.String("config"))
 		}
 
-		if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists {
+		err = r.Init(repo.FullNode)
+		if err != nil && err != repo.ErrRepoExists {
 			return xerrors.Errorf("repo init error: %w", err)
 		}
+		freshRepo := err != repo.ErrRepoExists
 
 		if !isLite {
 			if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil {
@@ -223,6 +233,15 @@ var DaemonCmd = &cli.Command{
 			genBytes = build.MaybeGenesis()
 		}
 
+		if cctx.IsSet("restore") {
+			if !freshRepo {
+				return xerrors.Errorf("restoring from backup is only possible with a fresh repo!")
+			}
+			if err := restore(cctx, r); err != nil {
+				return xerrors.Errorf("restoring from backup: %w", err)
+			}
+		}
+
 		chainfile := cctx.String("import-chain")
 		snapshot := cctx.String("import-snapshot")
 		if chainfile != "" || snapshot != "" {

cmd/tvx/codenames_test.go

@@ -18,7 +18,7 @@ func TestProtocolCodenames(t *testing.T) {
 		t.Fatal("expected breeze codename")
 	}
 
-	if height := build.UpgradeActorsV2Height + 1; GetProtocolCodename(height) != "actorsv2" {
+	if height := build.UpgradeActorsV2Height + 1; GetProtocolCodename(abi.ChainEpoch(height)) != "actorsv2" {
 		t.Fatal("expected actorsv2 codename")
 	}
 

cmd/tvx/exec.go

@@ -1,33 +1,48 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/fatih/color"
+	"github.com/filecoin-project/go-address"
+	cbornode "github.com/ipfs/go-ipld-cbor"
 	"github.com/urfave/cli/v2"
 
-	"github.com/filecoin-project/lotus/conformance"
-
 	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/conformance"
+	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 var execFlags struct {
 	file               string
+	out                string
+	driverOpts         cli.StringSlice
 	fallbackBlockstore bool
 }
 
+const (
+	optSaveBalances = "save-balances"
+)
+
 var execCmd = &cli.Command{
 	Name:        "exec",
-	Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream",
-	Action:      runExecLotus,
+	Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, a directory, or a ndjson stdin stream",
+	Action:      runExec,
 	Flags: []cli.Flag{
+		&repoFlag,
 		&cli.StringFlag{
 			Name:        "file",
-			Usage:       "input file; if not supplied, the vector will be read from stdin",
+			Usage:       "input file or directory; if not supplied, the vector will be read from stdin",
 			TakesFile:   true,
 			Destination: &execFlags.file,
 		},
@@ -36,10 +51,20 @@ var execCmd = &cli.Command{
 			Usage:       "sets the full node API as a fallback blockstore; use this if you're transplanting vectors and get block not found errors",
 			Destination: &execFlags.fallbackBlockstore,
 		},
+		&cli.StringFlag{
+			Name:        "out",
+			Usage:       "output directory where to save the results, only used when the input is a directory",
+			Destination: &execFlags.out,
+		},
+		&cli.StringSliceFlag{
+			Name:        "driver-opt",
+			Usage:       "comma-separated list of driver options (EXPERIMENTAL; will change), supported: 'save-balances=<dst>', 'pipeline-basefee' (unimplemented); only available in single-file mode",
+			Destination: &execFlags.driverOpts,
+		},
 	},
 }
 
-func runExecLotus(c *cli.Context) error {
+func runExec(c *cli.Context) error {
 	if execFlags.fallbackBlockstore {
 		if err := initialize(c); err != nil {
 			return fmt.Errorf("fallback blockstore was enabled, but could not resolve lotus API endpoint: %w", err)
@@ -48,30 +73,97 @@ func runExecLotus(c *cli.Context) error {
 		conformance.FallbackBlockstoreGetter = FullAPI
 	}
 
-	if file := execFlags.file; file != "" {
-		// we have a single test vector supplied as a file.
-		file, err := os.Open(file)
-		if err != nil {
-			return fmt.Errorf("failed to open test vector: %w", err)
-		}
-
-		var (
-			dec = json.NewDecoder(file)
-			tv  schema.TestVector
-		)
-
-		if err = dec.Decode(&tv); err != nil {
-			return fmt.Errorf("failed to decode test vector: %w", err)
-		}
-
-		return executeTestVector(tv)
+	path := execFlags.file
+	if path == "" {
+		return execVectorsStdin()
 	}
 
+	fi, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+
+	if fi.IsDir() {
+		// we're in directory mode; ensure the out directory exists.
+		outdir := execFlags.out
+		if outdir == "" {
+			return fmt.Errorf("no output directory provided")
+		}
+		if err := ensureDir(outdir); err != nil {
+			return err
+		}
+		return execVectorDir(path, outdir)
+	}
+
+	// process tipset vector options.
+	if err := processTipsetOpts(); err != nil {
+		return err
+	}
+
+	_, err = execVectorFile(new(conformance.LogReporter), path)
+	return err
+}
+
+func processTipsetOpts() error {
+	for _, opt := range execFlags.driverOpts.Value() {
+		switch ss := strings.Split(opt, "="); {
+		case ss[0] == optSaveBalances:
+			filename := ss[1]
+			log.Printf("saving balances after each tipset in: %s", filename)
+			balancesFile, err := os.Create(filename)
+			if err != nil {
+				return err
+			}
+			w := bufio.NewWriter(balancesFile)
+			cb := func(bs blockstore.Blockstore, params *conformance.ExecuteTipsetParams, res *conformance.ExecuteTipsetResult) {
+				cst := cbornode.NewCborStore(bs)
+				st, err := state.LoadStateTree(cst, res.PostStateRoot)
+				if err != nil {
+					return
+				}
+				_ = st.ForEach(func(addr address.Address, actor *types.Actor) error {
+					_, err := fmt.Fprintln(w, params.ExecEpoch, addr, actor.Balance)
+					return err
+				})
+				_ = w.Flush()
+			}
+			conformance.TipsetVectorOpts.OnTipsetApplied = append(conformance.TipsetVectorOpts.OnTipsetApplied, cb)
+
+		}
+
+	}
+	return nil
+}
+
+func execVectorDir(path string, outdir string) error {
+	files, err := filepath.Glob(filepath.Join(path, "*"))
+	if err != nil {
+		return fmt.Errorf("failed to glob input directory %s: %w", path, err)
+	}
+	for _, f := range files {
+		outfile := strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)) + ".out"
+		outpath := filepath.Join(outdir, outfile)
+		outw, err := os.Create(outpath)
+		if err != nil {
+			return fmt.Errorf("failed to create file %s: %w", outpath, err)
+		}
+
+		log.Printf("processing vector %s; sending output to %s", f, outpath)
+		log.SetOutput(io.MultiWriter(os.Stderr, outw)) // tee the output.
+		_, _ = execVectorFile(new(conformance.LogReporter), f)
+		log.SetOutput(os.Stderr)
+		_ = outw.Close()
+	}
+	return nil
+}
+
+func execVectorsStdin() error {
+	r := new(conformance.LogReporter)
 	for dec := json.NewDecoder(os.Stdin); ; {
 		var tv schema.TestVector
 		switch err := dec.Decode(&tv); err {
 		case nil:
-			if err = executeTestVector(tv); err != nil {
+			if _, err = executeTestVector(r, tv); err != nil {
 				return err
 			}
 		case io.EOF:
@@ -84,19 +176,30 @@ func runExecLotus(c *cli.Context) error {
 	}
 }
 
-func executeTestVector(tv schema.TestVector) error {
+func execVectorFile(r conformance.Reporter, path string) (diffs []string, error error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open test vector: %w", err)
+	}
+
+	var tv schema.TestVector
+	if err = json.NewDecoder(file).Decode(&tv); err != nil {
+		return nil, fmt.Errorf("failed to decode test vector: %w", err)
+	}
+	return executeTestVector(r, tv)
+}
+
+func executeTestVector(r conformance.Reporter, tv schema.TestVector) (diffs []string, err error) {
 	log.Println("executing test vector:", tv.Meta.ID)
 
 	for _, v := range tv.Pre.Variants {
-		r := new(conformance.LogReporter)
-
 		switch class, v := tv.Class, v; class {
 		case "message":
-			conformance.ExecuteMessageVector(r, &tv, &v)
+			diffs, err = conformance.ExecuteMessageVector(r, &tv, &v)
 		case "tipset":
-			conformance.ExecuteTipsetVector(r, &tv, &v)
+			diffs, err = conformance.ExecuteTipsetVector(r, &tv, &v)
 		default:
-			return fmt.Errorf("test vector class %s not supported", class)
+			return nil, fmt.Errorf("test vector class %s not supported", class)
 		}
 
 		if r.Failed() {
@@ -106,5 +209,5 @@ func executeTestVector(tv schema.TestVector) error {
 		}
 	}
 
-	return nil
+	return diffs, err
 }

cmd/tvx/extract.go

@@ -1,8 +1,14 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
 
+	"github.com/filecoin-project/test-vectors/schema"
 	"github.com/urfave/cli/v2"
 )
 
@@ -21,6 +27,7 @@ type extractOpts struct {
 	retain             string
 	precursor          string
 	ignoreSanityChecks bool
+	squash             bool
 }
 
 var extractFlags extractOpts
@@ -62,13 +69,13 @@ var extractCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:        "tsk",
-			Usage:       "tipset key to extract into a vector",
+			Usage:       "tipset key to extract into a vector, or range of tipsets in tsk1..tsk2 form",
 			Destination: &extractFlags.tsk,
 		},
 		&cli.StringFlag{
 			Name:        "out",
 			Aliases:     []string{"o"},
-			Usage:       "file to write test vector to",
+			Usage:       "file to write test vector to, or directory to write the batch to",
 			Destination: &extractFlags.file,
 		},
 		&cli.StringFlag{
@@ -93,6 +100,12 @@ var extractCmd = &cli.Command{
 			Value:       false,
 			Destination: &extractFlags.ignoreSanityChecks,
 		},
+		&cli.BoolFlag{
+			Name:        "squash",
+			Usage:       "when extracting a tipset range, squash all tipsets into a single vector",
+			Value:       false,
+			Destination: &extractFlags.squash,
+		},
 	},
 }
 
@@ -106,3 +119,43 @@ func runExtract(_ *cli.Context) error {
 		return fmt.Errorf("unsupported vector class")
 	}
 }
+
+// writeVector writes the vector into the specified file, or to stdout if
+// file is empty.
+func writeVector(vector *schema.TestVector, file string) (err error) {
+	output := io.WriteCloser(os.Stdout)
+	if file := file; file != "" {
+		dir := filepath.Dir(file)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("unable to create directory %s: %w", dir, err)
+		}
+		output, err = os.Create(file)
+		if err != nil {
+			return err
+		}
+		defer output.Close() //nolint:errcheck
+		defer log.Printf("wrote test vector to file: %s", file)
+	}
+
+	enc := json.NewEncoder(output)
+	enc.SetIndent("", "  ")
+	return enc.Encode(&vector)
+}
+
+// writeVectors writes each vector to a different file under the specified
+// directory.
+func writeVectors(dir string, vectors ...*schema.TestVector) error {
+	// verify the output directory exists.
+	if err := ensureDir(dir); err != nil {
+		return err
+	}
+	// write each vector to its file.
+	for _, v := range vectors {
+		id := v.Meta.ID
+		path := filepath.Join(dir, fmt.Sprintf("%s.json", id))
+		if err := writeVector(v, path); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/tvx/extract_message.go

@@ -4,12 +4,9 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"log"
-	"os"
-	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/filecoin-project/go-address"
@@ -316,28 +313,7 @@ func doExtractMessage(opts extractOpts) error {
 			},
 		},
 	}
-
-	return writeVector(vector, opts.file)
-}
-
-func writeVector(vector schema.TestVector, file string) (err error) {
-	output := io.WriteCloser(os.Stdout)
-	if file := file; file != "" {
-		dir := filepath.Dir(file)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			return fmt.Errorf("unable to create directory %s: %w", dir, err)
-		}
-		output, err = os.Create(file)
-		if err != nil {
-			return err
-		}
-		defer output.Close() //nolint:errcheck
-		defer log.Printf("wrote test vector to file: %s", file)
-	}
-
-	enc := json.NewEncoder(output)
-	enc.SetIndent("", "  ")
-	return enc.Encode(&vector)
+	return writeVector(&vector, opts.file)
 }
 
 // resolveFromChain queries the chain for the provided message, using the block CID to

cmd/tvx/extract_tipset.go

@@ -6,10 +6,12 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 
 	"github.com/filecoin-project/test-vectors/schema"
 	"github.com/ipfs/go-cid"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/conformance"
 )
@@ -17,170 +19,259 @@ import (
 func doExtractTipset(opts extractOpts) error {
 	ctx := context.Background()
 
-	if opts.tsk == "" {
-		return fmt.Errorf("tipset key cannot be empty")
-	}
-
 	if opts.retain != "accessed-cids" {
 		return fmt.Errorf("tipset extraction only supports 'accessed-cids' state retention")
 	}
 
-	ts, err := lcli.ParseTipSetRef(ctx, FullAPI, opts.tsk)
-	if err != nil {
-		return fmt.Errorf("failed to fetch tipset: %w", err)
+	if opts.tsk == "" {
+		return fmt.Errorf("tipset key cannot be empty")
 	}
 
-	log.Printf("tipset block count: %d", len(ts.Blocks()))
-
-	var blocks []schema.Block
-	for _, b := range ts.Blocks() {
-		msgs, err := FullAPI.ChainGetBlockMessages(ctx, b.Cid())
+	ss := strings.Split(opts.tsk, "..")
+	switch len(ss) {
+	case 1: // extracting a single tipset.
+		ts, err := lcli.ParseTipSetRef(ctx, FullAPI, opts.tsk)
 		if err != nil {
-			return fmt.Errorf("failed to get block messages (cid: %s): %w", b.Cid(), err)
+			return fmt.Errorf("failed to fetch tipset: %w", err)
+		}
+		v, err := extractTipsets(ctx, ts)
+		if err != nil {
+			return err
+		}
+		return writeVector(v, opts.file)
+
+	case 2: // extracting a range of tipsets.
+		left, err := lcli.ParseTipSetRef(ctx, FullAPI, ss[0])
+		if err != nil {
+			return fmt.Errorf("failed to fetch tipset %s: %w", ss[0], err)
+		}
+		right, err := lcli.ParseTipSetRef(ctx, FullAPI, ss[1])
+		if err != nil {
+			return fmt.Errorf("failed to fetch tipset %s: %w", ss[1], err)
 		}
 
-		log.Printf("block %s has %d messages", b.Cid(), len(msgs.Cids))
+		// resolve the tipset range.
+		tss, err := resolveTipsetRange(ctx, left, right)
+		if err != nil {
+			return err
+		}
 
-		packed := make([]schema.Base64EncodedBytes, 0, len(msgs.Cids))
-		for _, m := range msgs.BlsMessages {
-			b, err := m.Serialize()
+		// are are squashing all tipsets into a single multi-tipset vector?
+		if opts.squash {
+			vector, err := extractTipsets(ctx, tss...)
 			if err != nil {
-				return fmt.Errorf("failed to serialize message: %w", err)
+				return err
 			}
-			packed = append(packed, b)
+			return writeVector(vector, opts.file)
 		}
-		for _, m := range msgs.SecpkMessages {
-			b, err := m.Message.Serialize()
-			if err != nil {
-				return fmt.Errorf("failed to serialize message: %w", err)
-			}
-			packed = append(packed, b)
+
+		// we are generating a single-tipset vector per tipset.
+		vectors, err := extractIndividualTipsets(ctx, tss...)
+		if err != nil {
+			return err
 		}
-		blocks = append(blocks, schema.Block{
-			MinerAddr: b.Miner,
-			WinCount:  b.ElectionProof.WinCount,
-			Messages:  packed,
-		})
+		return writeVectors(opts.file, vectors...)
+
+	default:
+		return fmt.Errorf("unrecognized tipset format")
 	}
+}
 
+func resolveTipsetRange(ctx context.Context, left *types.TipSet, right *types.TipSet) (tss []*types.TipSet, err error) {
+	// start from the right tipset and walk back the chain until the left tipset, inclusive.
+	for curr := right; curr.Key() != left.Parents(); {
+		tss = append(tss, curr)
+		curr, err = FullAPI.ChainGetTipSet(ctx, curr.Parents())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get tipset %s (height: %d): %w", curr.Parents(), curr.Height()-1, err)
+		}
+	}
+	// reverse the slice.
+	for i, j := 0, len(tss)-1; i < j; i, j = i+1, j-1 {
+		tss[i], tss[j] = tss[j], tss[i]
+	}
+	return tss, nil
+}
+
+func extractIndividualTipsets(ctx context.Context, tss ...*types.TipSet) (vectors []*schema.TestVector, err error) {
+	for _, ts := range tss {
+		v, err := extractTipsets(ctx, ts)
+		if err != nil {
+			return nil, err
+		}
+		vectors = append(vectors, v)
+	}
+	return vectors, nil
+}
+
+func extractTipsets(ctx context.Context, tss ...*types.TipSet) (*schema.TestVector, error) {
 	var (
 		// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
 		pst = NewProxyingStores(ctx, FullAPI)
 		g   = NewSurgeon(ctx, FullAPI, pst)
+
+		// recordingRand will record randomness so we can embed it in the test vector.
+		recordingRand = conformance.NewRecordingRand(new(conformance.LogReporter), FullAPI)
 	)
 
+	tbs, ok := pst.Blockstore.(TracingBlockstore)
+	if !ok {
+		return nil, fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
+	}
+
 	driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{
 		DisableVMFlush: true,
 	})
 
+	base := tss[0]
+	last := tss[len(tss)-1]
+
 	// this is the root of the state tree we start with.
-	root := ts.ParentState()
+	root := base.ParentState()
 	log.Printf("base state tree root CID: %s", root)
 
-	basefee := ts.Blocks()[0].ParentBaseFee
-	log.Printf("basefee: %s", basefee)
-
-	tipset := schema.Tipset{
-		BaseFee: *basefee.Int,
-		Blocks:  blocks,
+	codename := GetProtocolCodename(base.Height())
+	nv, err := FullAPI.StateNetworkVersion(ctx, base.Key())
+	if err != nil {
+		return nil, err
 	}
 
-	// recordingRand will record randomness so we can embed it in the test vector.
-	recordingRand := conformance.NewRecordingRand(new(conformance.LogReporter), FullAPI)
+	version, err := FullAPI.Version(ctx)
+	if err != nil {
+		return nil, err
+	}
 
-	log.Printf("using state retention strategy: %s", extractFlags.retain)
+	ntwkName, err := FullAPI.StateNetworkName(ctx)
+	if err != nil {
+		return nil, err
+	}
 
-	tbs, ok := pst.Blockstore.(TracingBlockstore)
-	if !ok {
-		return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
+	vector := schema.TestVector{
+		Class: schema.ClassTipset,
+		Meta: &schema.Metadata{
+			ID: fmt.Sprintf("@%d..@%d", base.Height(), last.Height()),
+			Gen: []schema.GenerationData{
+				{Source: fmt.Sprintf("network:%s", ntwkName)},
+				{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
+			// will be completed by extra tipset stamps.
+		},
+		Selector: schema.Selector{
+			schema.SelectorMinProtocolVersion: codename,
+		},
+		Pre: &schema.Preconditions{
+			Variants: []schema.Variant{
+				{ID: codename, Epoch: int64(base.Height()), NetworkVersion: uint(nv)},
+			},
+			StateTree: &schema.StateTree{
+				RootCID: base.ParentState(),
+			},
+		},
+		Post: &schema.Postconditions{
+			StateTree: new(schema.StateTree),
+		},
 	}
 
 	tbs.StartTracing()
 
-	params := conformance.ExecuteTipsetParams{
-		Preroot:     ts.ParentState(),
-		ParentEpoch: ts.Height() - 1,
-		Tipset:      &tipset,
-		ExecEpoch:   ts.Height(),
-		Rand:        recordingRand,
-	}
-	result, err := driver.ExecuteTipset(pst.Blockstore, pst.Datastore, params)
-	if err != nil {
-		return fmt.Errorf("failed to execute tipset: %w", err)
+	roots := []cid.Cid{base.ParentState()}
+	for i, ts := range tss {
+		log.Printf("tipset %s block count: %d", ts.Key(), len(ts.Blocks()))
+
+		var blocks []schema.Block
+		for _, b := range ts.Blocks() {
+			msgs, err := FullAPI.ChainGetBlockMessages(ctx, b.Cid())
+			if err != nil {
+				return nil, fmt.Errorf("failed to get block messages (cid: %s): %w", b.Cid(), err)
+			}
+
+			log.Printf("block %s has %d messages", b.Cid(), len(msgs.Cids))
+
+			packed := make([]schema.Base64EncodedBytes, 0, len(msgs.Cids))
+			for _, m := range msgs.BlsMessages {
+				b, err := m.Serialize()
+				if err != nil {
+					return nil, fmt.Errorf("failed to serialize message: %w", err)
+				}
+				packed = append(packed, b)
+			}
+			for _, m := range msgs.SecpkMessages {
+				b, err := m.Message.Serialize()
+				if err != nil {
+					return nil, fmt.Errorf("failed to serialize message: %w", err)
+				}
+				packed = append(packed, b)
+			}
+			blocks = append(blocks, schema.Block{
+				MinerAddr: b.Miner,
+				WinCount:  b.ElectionProof.WinCount,
+				Messages:  packed,
+			})
+		}
+
+		basefee := base.Blocks()[0].ParentBaseFee
+		log.Printf("tipset basefee: %s", basefee)
+
+		tipset := schema.Tipset{
+			BaseFee:     *basefee.Int,
+			Blocks:      blocks,
+			EpochOffset: int64(i),
+		}
+
+		params := conformance.ExecuteTipsetParams{
+			Preroot:     roots[len(roots)-1],
+			ParentEpoch: ts.Height() - 1,
+			Tipset:      &tipset,
+			ExecEpoch:   ts.Height(),
+			Rand:        recordingRand,
+		}
+
+		result, err := driver.ExecuteTipset(pst.Blockstore, pst.Datastore, params)
+		if err != nil {
+			return nil, fmt.Errorf("failed to execute tipset: %w", err)
+		}
+
+		roots = append(roots, result.PostStateRoot)
+
+		// update the vector.
+		vector.ApplyTipsets = append(vector.ApplyTipsets, tipset)
+		vector.Post.ReceiptsRoots = append(vector.Post.ReceiptsRoots, result.ReceiptsRoot)
+
+		for _, res := range result.AppliedResults {
+			vector.Post.Receipts = append(vector.Post.Receipts, &schema.Receipt{
+				ExitCode:    int64(res.ExitCode),
+				ReturnValue: res.Return,
+				GasUsed:     res.GasUsed,
+			})
+		}
+
+		vector.Meta.Gen = append(vector.Meta.Gen, schema.GenerationData{
+			Source: "tipset:" + ts.Key().String(),
+		})
 	}
 
 	accessed := tbs.FinishTracing()
 
+	//
+	// ComputeBaseFee(ctx, baseTs)
+
 	// write a CAR with the accessed state into a buffer.
 	var (
 		out = new(bytes.Buffer)
 		gw  = gzip.NewWriter(out)
 	)
-	if err := g.WriteCARIncluding(gw, accessed, ts.ParentState(), result.PostStateRoot); err != nil {
-		return err
+	if err := g.WriteCARIncluding(gw, accessed, roots...); err != nil {
+		return nil, err
 	}
 	if err = gw.Flush(); err != nil {
-		return err
+		return nil, err
 	}
 	if err = gw.Close(); err != nil {
-		return err
+		return nil, err
 	}
 
-	codename := GetProtocolCodename(ts.Height())
-	nv, err := FullAPI.StateNetworkVersion(ctx, ts.Key())
-	if err != nil {
-		return err
-	}
+	vector.Randomness = recordingRand.Recorded()
+	vector.Post.StateTree.RootCID = roots[len(roots)-1]
+	vector.CAR = out.Bytes()
 
-	version, err := FullAPI.Version(ctx)
-	if err != nil {
-		return err
-	}
-
-	ntwkName, err := FullAPI.StateNetworkName(ctx)
-	if err != nil {
-		return err
-	}
-
-	vector := schema.TestVector{
-		Class: schema.ClassTipset,
-		Meta: &schema.Metadata{
-			ID: opts.id,
-			Gen: []schema.GenerationData{
-				{Source: fmt.Sprintf("network:%s", ntwkName)},
-				{Source: fmt.Sprintf("tipset:%s", ts.Key())},
-				{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
-		},
-		Selector: schema.Selector{
-			schema.SelectorMinProtocolVersion: codename,
-		},
-		Randomness: recordingRand.Recorded(),
-		CAR:        out.Bytes(),
-		Pre: &schema.Preconditions{
-			Variants: []schema.Variant{
-				{ID: codename, Epoch: int64(ts.Height()), NetworkVersion: uint(nv)},
-			},
-			BaseFee: basefee.Int,
-			StateTree: &schema.StateTree{
-				RootCID: ts.ParentState(),
-			},
-		},
-		ApplyTipsets: []schema.Tipset{tipset},
-		Post: &schema.Postconditions{
-			StateTree: &schema.StateTree{
-				RootCID: result.PostStateRoot,
-			},
-			ReceiptsRoots: []cid.Cid{result.ReceiptsRoot},
-		},
-	}
-
-	for _, res := range result.AppliedResults {
-		vector.Post.Receipts = append(vector.Post.Receipts, &schema.Receipt{
-			ExitCode:    int64(res.ExitCode),
-			ReturnValue: res.Return,
-			GasUsed:     res.GasUsed,
-		})
-	}
-
-	return writeVector(vector, opts.file)
+	return &vector, nil
 }

cmd/tvx/main.go

@@ -113,3 +113,19 @@ func destroy(_ *cli.Context) error {
 	}
 	return nil
 }
+
+func ensureDir(path string) error {
+	switch fi, err := os.Stat(path); {
+	case os.IsNotExist(err):
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %w", path, err)
+		}
+	case err == nil:
+		if !fi.IsDir() {
+			return fmt.Errorf("path %s is not a directory: %w", path, err)
+		}
+	default:
+		return fmt.Errorf("failed to stat directory %s: %w", path, err)
+	}
+	return nil
+}

cmd/tvx/simulate.go

@@ -202,7 +202,7 @@ func runSimulateCmd(_ *cli.Context) error {
 		},
 	}
 
-	if err := writeVector(vector, simulateFlags.out); err != nil {
+	if err := writeVector(&vector, simulateFlags.out); err != nil {
 		return fmt.Errorf("failed to write vector: %w", err)
 	}
 

cmd/tvx/stores.go

@@ -149,3 +149,14 @@ func (pb *proxyingBlockstore) Put(block blocks.Block) error {
 	pb.lk.Unlock()
 	return pb.Blockstore.Put(block)
 }
+
+func (pb *proxyingBlockstore) PutMany(blocks []blocks.Block) error {
+	pb.lk.Lock()
+	if pb.tracing {
+		for _, b := range blocks {
+			pb.traced[b.Cid()] = struct{}{}
+		}
+	}
+	pb.lk.Unlock()
+	return pb.Blockstore.PutMany(blocks)
+}

conformance/corpus_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/filecoin-project/test-vectors/schema"
 )
 
-var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant){
+var invokees = map[schema.Class]func(Reporter, *schema.TestVector, *schema.Variant) ([]string, error){
 	schema.ClassMessage: ExecuteMessageVector,
 	schema.ClassTipset:  ExecuteTipsetVector,
 }
@@ -133,7 +133,7 @@ func TestConformance(t *testing.T) {
 			for _, variant := range vector.Pre.Variants {
 				variant := variant
 				t.Run(variant.ID, func(t *testing.T) {
-					invokee(t, &vector, &variant)
+					_, _ = invokee(t, &vector, &variant) //nolint:errcheck
 				})
 			}
 		})

conformance/driver.go

@@ -71,6 +71,9 @@ type ExecuteTipsetResult struct {
 	AppliedMessages []*types.Message
 	// AppliedResults stores the results of AppliedMessages, in the same order.
 	AppliedResults []*vm.ApplyRet
+
+	// PostBaseFee returns the basefee after applying this tipset.
+	PostBaseFee abi.TokenAmount
 }
 
 type ExecuteTipsetParams struct {

conformance/runner.go

@@ -14,6 +14,7 @@ import (
 	"github.com/fatih/color"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/hashicorp/go-multierror"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
@@ -38,8 +39,19 @@ var FallbackBlockstoreGetter interface {
 	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 }
 
+var TipsetVectorOpts struct {
+	// PipelineBaseFee pipelines the basefee in multi-tipset vectors from one
+	// tipset to another. Basefees in the vector are ignored, except for that of
+	// the first tipset. UNUSED.
+	PipelineBaseFee bool
+
+	// OnTipsetApplied contains callback functions called after a tipset has been
+	// applied.
+	OnTipsetApplied []func(bs blockstore.Blockstore, params *ExecuteTipsetParams, res *ExecuteTipsetResult)
+}
+
 // ExecuteMessageVector executes a message-class test vector.
-func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) {
+func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) (diffs []string, err error) {
 	var (
 		ctx       = context.Background()
 		baseEpoch = variant.Epoch
@@ -88,14 +100,16 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector, variant *schema
 	// Once all messages are applied, assert that the final state root matches
 	// the expected postcondition root.
 	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
-		r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
-		dumpThreeWayStateDiff(r, vector, bs, root)
-		r.FailNow()
+		ierr := fmt.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
+		r.Errorf(ierr.Error())
+		err = multierror.Append(err, ierr)
+		diffs = dumpThreeWayStateDiff(r, vector, bs, root)
 	}
+	return diffs, err
 }
 
 // ExecuteTipsetVector executes a tipset-class test vector.
-func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) {
+func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.Variant) (diffs []string, err error) {
 	var (
 		ctx       = context.Background()
 		baseEpoch = abi.ChainEpoch(variant.Epoch)
@@ -107,6 +121,7 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.
 	bs, err := LoadBlockstore(vector.CAR)
 	if err != nil {
 		r.Fatalf("failed to load the vector CAR: %w", err)
+		return nil, err
 	}
 
 	// Create a new Driver.
@@ -118,15 +133,22 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.
 	for i, ts := range vector.ApplyTipsets {
 		ts := ts // capture
 		execEpoch := baseEpoch + abi.ChainEpoch(ts.EpochOffset)
-		ret, err := driver.ExecuteTipset(bs, tmpds, ExecuteTipsetParams{
+		params := ExecuteTipsetParams{
 			Preroot:     root,
 			ParentEpoch: prevEpoch,
 			Tipset:      &ts,
 			ExecEpoch:   execEpoch,
 			Rand:        NewReplayingRand(r, vector.Randomness),
-		})
+		}
+		ret, err := driver.ExecuteTipset(bs, tmpds, params)
 		if err != nil {
 			r.Fatalf("failed to apply tipset %d: %s", i, err)
+			return nil, err
+		}
+
+		// invoke callbacks.
+		for _, cb := range TipsetVectorOpts.OnTipsetApplied {
+			cb(bs, &params, ret)
 		}
 
 		for j, v := range ret.AppliedResults {
@@ -136,7 +158,9 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.
 
 		// Compare the receipts root.
 		if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
-			r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
+			ierr := fmt.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
+			r.Errorf(ierr.Error())
+			err = multierror.Append(err, ierr)
 		}
 
 		prevEpoch = execEpoch
@@ -146,10 +170,12 @@ func ExecuteTipsetVector(r Reporter, vector *schema.TestVector, variant *schema.
 	// Once all messages are applied, assert that the final state root matches
 	// the expected postcondition root.
 	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
-		r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
-		dumpThreeWayStateDiff(r, vector, bs, root)
-		r.FailNow()
+		ierr := fmt.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
+		r.Errorf(ierr.Error())
+		err = multierror.Append(err, ierr)
+		diffs = dumpThreeWayStateDiff(r, vector, bs, root)
 	}
+	return diffs, err
 }
 
 // AssertMsgResult compares a message result. It takes the expected receipt
@@ -169,7 +195,7 @@ func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet,
 	}
 }
 
-func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
+func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) []string {
 	// check if statediff exists; if not, skip.
 	if err := exec.Command("statediff", "--help").Run(); err != nil {
 		r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
@@ -178,7 +204,7 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.
 		r.Log("$ cd statediff")
 		r.Log("$ go generate ./...")
 		r.Log("$ go install ./cmd/statediff")
-		return
+		return nil
 	}
 
 	tmpCar, err := writeStateToTempCAR(bs,
@@ -188,6 +214,7 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.
 	)
 	if err != nil {
 		r.Fatalf("failed to write temporary state CAR: %s", err)
+		return nil
 	}
 	defer os.RemoveAll(tmpCar) //nolint:errcheck
 
@@ -202,28 +229,43 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.
 		d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
 	)
 
-	printDiff := func(left, right cid.Cid) {
+	diff := func(left, right cid.Cid) string {
 		cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
 		b, err := cmd.CombinedOutput()
 		if err != nil {
 			r.Fatalf("statediff failed: %s", err)
 		}
-		r.Log(string(b))
+		return string(b)
 	}
 
 	bold := color.New(color.Bold).SprintfFunc()
 
+	r.Log(bold("-----BEGIN STATEDIFF-----"))
+
 	// run state diffs.
 	r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
 
 	r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
-	printDiff(vector.Post.StateTree.RootCID, actual)
+	diffA := diff(vector.Post.StateTree.RootCID, actual)
+	r.Log(bold("----------BEGIN STATEDIFF A----------"))
+	r.Log(diffA)
+	r.Log(bold("----------END STATEDIFF A----------"))
 
 	r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
-	printDiff(vector.Pre.StateTree.RootCID, actual)
+	diffB := diff(vector.Pre.StateTree.RootCID, actual)
+	r.Log(bold("----------BEGIN STATEDIFF B----------"))
+	r.Log(diffB)
+	r.Log(bold("----------END STATEDIFF B----------"))
 
 	r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
-	printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
+	diffC := diff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
+	r.Log(bold("----------BEGIN STATEDIFF C----------"))
+	r.Log(diffC)
+	r.Log(bold("----------END STATEDIFF C----------"))
+
+	r.Log(bold("-----END STATEDIFF-----"))
+
+	return []string{diffA, diffB, diffC}
 }
 
 // writeStateToTempCAR writes the provided roots to a temporary CAR that'll be

documentation/en/api-methods-miner.md

@@ -99,6 +99,9 @@
   * [SectorSetExpectedSealDuration](#SectorSetExpectedSealDuration)
   * [SectorSetSealDelay](#SectorSetSealDelay)
   * [SectorStartSealing](#SectorStartSealing)
+  * [SectorTerminate](#SectorTerminate)
+  * [SectorTerminateFlush](#SectorTerminateFlush)
+  * [SectorTerminatePending](#SectorTerminatePending)
 * [Sectors](#Sectors)
   * [SectorsList](#SectorsList)
   * [SectorsListInStates](#SectorsListInStates)
@@ -193,7 +196,8 @@ Response:
 ```json
 {
   "PreCommitControl": null,
-  "CommitControl": null
+  "CommitControl": null,
+  "TerminateControl": null
 }
 ```
 
@@ -1475,7 +1479,9 @@ Inputs:
 Response: `{}`
 
 ### SectorRemove
-There are not yet any comments for this method.
+SectorRemove removes the sector from storage. It doesn't terminate it on-chain, which can
+be done with SectorTerminate. Removing and not terminating live sectors will cause additional penalties.
+
 
 Perms: admin
 
@@ -1535,6 +1541,43 @@ Inputs:
 
 Response: `{}`
 
+### SectorTerminate
+SectorTerminate terminates the sector on-chain (adding it to a termination batch first), then
+automatically removes it from storage
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  9
+]
+```
+
+Response: `{}`
+
+### SectorTerminateFlush
+SectorTerminateFlush immediately sends a terminate message with sectors batched for termination.
+Returns null if message wasn't sent
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `null`
+
+### SectorTerminatePending
+SectorTerminatePending returns a list of pending sector terminations to be sent in the next batch message
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `null`
+
 ## Sectors
 
 

documentation/en/api-methods.md

@@ -68,6 +68,8 @@
   * [LogList](#LogList)
   * [LogSetLevel](#LogSetLevel)
 * [Market](#Market)
+  * [MarketAddBalance](#MarketAddBalance)
+  * [MarketGetReserved](#MarketGetReserved)
   * [MarketReleaseFunds](#MarketReleaseFunds)
   * [MarketReserveFunds](#MarketReserveFunds)
   * [MarketWithdraw](#MarketWithdraw)
@@ -175,6 +177,7 @@
   * [StateReadState](#StateReadState)
   * [StateReplay](#StateReplay)
   * [StateSearchMsg](#StateSearchMsg)
+  * [StateSearchMsgLimited](#StateSearchMsgLimited)
   * [StateSectorExpiration](#StateSectorExpiration)
   * [StateSectorGetInfo](#StateSectorGetInfo)
   * [StateSectorPartition](#StateSectorPartition)
@@ -1653,6 +1656,43 @@ Response: `{}`
 ## Market
 
 
+### MarketAddBalance
+MarketAddBalance adds funds to the market actor
+
+
+Perms: sign
+
+Inputs:
+```json
+[
+  "f01234",
+  "f01234",
+  "0"
+]
+```
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
+### MarketGetReserved
+MarketGetReserved gets the amount of funds that are currently reserved for the address
+
+
+Perms: sign
+
+Inputs:
+```json
+[
+  "f01234"
+]
+```
+
+Response: `"0"`
+
 ### MarketReleaseFunds
 MarketReleaseFunds releases funds reserved by MarketReserveFunds
 
@@ -4308,7 +4348,7 @@ Inputs:
 ]
 ```
 
-Response: `8`
+Response: `9`
 
 ### StateReadState
 StateReadState returns the indicated actor's state.
@@ -4471,6 +4511,46 @@ Response:
 }
 ```
 
+### StateSearchMsgLimited
+StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  },
+  10101
+]
+```
+
+Response:
+```json
+{
+  "Message": {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  },
+  "Receipt": {
+    "ExitCode": 0,
+    "Return": "Ynl0ZSBhcnJheQ==",
+    "GasUsed": 9
+  },
+  "ReturnDec": {},
+  "TipSet": [
+    {
+      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+    },
+    {
+      "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve"
+    }
+  ],
+  "Height": 10101
+}
+```
+
 ### StateSectorExpiration
 StateSectorExpiration returns epoch at which given sector will expire
 

extern/sector-storage/ffiwrapper/sealer_cgo.go

@@ -45,6 +45,10 @@ func (sb *Sealer) NewSector(ctx context.Context, sector storage.SectorRef) error
 }
 
 func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) {
+	// TODO: allow tuning those:
+	chunk := abi.PaddedPieceSize(4 << 20)
+	parallel := runtime.NumCPU()
+
 	var offset abi.UnpaddedPieceSize
 	for _, size := range existingPieceSizes {
 		offset += size
@@ -108,10 +112,16 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
 
 	pr := io.TeeReader(io.LimitReader(file, int64(pieceSize)), pw)
 
-	chunk := abi.PaddedPieceSize(4 << 20)
+	throttle := make(chan []byte, parallel)
+	piecePromises := make([]func() (abi.PieceInfo, error), 0)
 
 	buf := make([]byte, chunk.Unpadded())
-	var pieceCids []abi.PieceInfo
+	for i := 0; i < parallel; i++ {
+		if abi.UnpaddedPieceSize(i)*chunk.Unpadded() >= pieceSize {
+			break // won't use this many buffers
+		}
+		throttle <- make([]byte, chunk.Unpadded())
+	}
 
 	for {
 		var read int
@@ -132,13 +142,39 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
 			break
 		}
 
-		c, err := sb.pieceCid(sector.ProofType, buf[:read])
-		if err != nil {
-			return abi.PieceInfo{}, xerrors.Errorf("pieceCid error: %w", err)
-		}
-		pieceCids = append(pieceCids, abi.PieceInfo{
-			Size:     abi.UnpaddedPieceSize(len(buf[:read])).Padded(),
-			PieceCID: c,
+		done := make(chan struct {
+			cid.Cid
+			error
+		}, 1)
+		pbuf := <-throttle
+		copy(pbuf, buf[:read])
+
+		go func(read int) {
+			defer func() {
+				throttle <- pbuf
+			}()
+
+			c, err := sb.pieceCid(sector.ProofType, pbuf[:read])
+			done <- struct {
+				cid.Cid
+				error
+			}{c, err}
+		}(read)
+
+		piecePromises = append(piecePromises, func() (abi.PieceInfo, error) {
+			select {
+			case e := <-done:
+				if e.error != nil {
+					return abi.PieceInfo{}, e.error
+				}
+
+				return abi.PieceInfo{
+					Size:     abi.UnpaddedPieceSize(len(buf[:read])).Padded(),
+					PieceCID: e.Cid,
+				}, nil
+			case <-ctx.Done():
+				return abi.PieceInfo{}, ctx.Err()
+			}
 		})
 	}
 
@@ -155,8 +191,16 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi
 	}
 	stagedFile = nil
 
-	if len(pieceCids) == 1 {
-		return pieceCids[0], nil
+	if len(piecePromises) == 1 {
+		return piecePromises[0]()
+	}
+
+	pieceCids := make([]abi.PieceInfo, len(piecePromises))
+	for i, promise := range piecePromises {
+		pieceCids[i], err = promise()
+		if err != nil {
+			return abi.PieceInfo{}, err
+		}
 	}
 
 	pieceCID, err := ffi.GenerateUnsealedCID(sector.ProofType, pieceCids)

extern/sector-storage/ffiwrapper/sealer_test.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper/basicfs"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
 )
 
 func init() {
@@ -622,3 +623,89 @@ func TestGenerateUnsealedCID(t *testing.T) {
 		[][]byte{barr(1, 16), barr(0, 16), barr(2, 8), barr(3, 16), barr(0, 16), barr(0, 8), barr(4, 4), barr(5, 16), barr(0, 16), barr(0, 8)},
 	)
 }
+
+func TestAddPiece512M(t *testing.T) {
+	sz := abi.PaddedPieceSize(512 << 20).Unpadded()
+
+	cdir, err := ioutil.TempDir("", "sbtest-c-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	miner := abi.ActorID(123)
+
+	sp := &basicfs.Provider{
+		Root: cdir,
+	}
+	sb, err := New(sp)
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+	cleanup := func() {
+		if t.Failed() {
+			fmt.Printf("not removing %s\n", cdir)
+			return
+		}
+		if err := os.RemoveAll(cdir); err != nil {
+			t.Error(err)
+		}
+	}
+	t.Cleanup(cleanup)
+
+	r := rand.New(rand.NewSource(0x7e5))
+
+	c, err := sb.AddPiece(context.TODO(), storage.SectorRef{
+		ID: abi.SectorID{
+			Miner:  miner,
+			Number: 0,
+		},
+		ProofType: abi.RegisteredSealProof_StackedDrg512MiBV1_1,
+	}, nil, sz, io.LimitReader(r, int64(sz)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	require.Equal(t, "baga6ea4seaqhyticusemlcrjhvulpfng4nint6bu3wpe5s3x4bnuj2rs47hfacy", c.PieceCID.String())
+}
+
+func BenchmarkAddPiece512M(b *testing.B) {
+	sz := abi.PaddedPieceSize(512 << 20).Unpadded()
+	b.SetBytes(int64(sz))
+
+	cdir, err := ioutil.TempDir("", "sbtest-c-")
+	if err != nil {
+		b.Fatal(err)
+	}
+	miner := abi.ActorID(123)
+
+	sp := &basicfs.Provider{
+		Root: cdir,
+	}
+	sb, err := New(sp)
+	if err != nil {
+		b.Fatalf("%+v", err)
+	}
+	cleanup := func() {
+		if b.Failed() {
+			fmt.Printf("not removing %s\n", cdir)
+			return
+		}
+		if err := os.RemoveAll(cdir); err != nil {
+			b.Error(err)
+		}
+	}
+	b.Cleanup(cleanup)
+
+	for i := 0; i < b.N; i++ {
+		c, err := sb.AddPiece(context.TODO(), storage.SectorRef{
+			ID: abi.SectorID{
+				Miner:  miner,
+				Number: abi.SectorNumber(i),
+			},
+			ProofType: abi.RegisteredSealProof_StackedDrg512MiBV1_1,
+		}, nil, sz, io.LimitReader(&nullreader.Reader{}, int64(sz)))
+		if err != nil {
+			b.Fatal(err)
+		}
+		fmt.Println(c)
+	}
+}

extern/sector-storage/fsutil/filesize_unix.go

@@ -2,6 +2,7 @@ package fsutil
 
 import (
 	"os"
+	"path/filepath"
 	"syscall"
 
 	"golang.org/x/xerrors"
@@ -11,19 +12,32 @@ type SizeInfo struct {
 	OnDisk int64
 }
 
-// FileSize returns bytes used by a file on disk
+// FileSize returns bytes used by a file or directory on disk
+// NOTE: We care about the allocated bytes, not file or directory size
 func FileSize(path string) (SizeInfo, error) {
-	var stat syscall.Stat_t
-	if err := syscall.Stat(path, &stat); err != nil {
-		if err == syscall.ENOENT {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			stat, ok := info.Sys().(*syscall.Stat_t)
+			if !ok {
+				return xerrors.New("FileInfo.Sys of wrong type")
+			}
+
+			// NOTE: stat.Blocks is in 512B blocks, NOT in stat.Blksize		return SizeInfo{size}, nil
+			//  See https://www.gnu.org/software/libc/manual/html_node/Attribute-Meanings.html
+			size += int64(stat.Blocks) * 512 // nolint NOTE: int64 cast is needed on osx
+		}
+		return err
+	})
+	if err != nil {
+		if os.IsNotExist(err) {
 			return SizeInfo{}, os.ErrNotExist
 		}
-		return SizeInfo{}, xerrors.Errorf("stat: %w", err)
+		return SizeInfo{}, xerrors.Errorf("filepath.Walk err: %w", err)
 	}
 
-	// NOTE: stat.Blocks is in 512B blocks, NOT in stat.Blksize
-	//  See https://www.gnu.org/software/libc/manual/html_node/Attribute-Meanings.html
-	return SizeInfo{
-		int64(stat.Blocks) * 512, // nolint NOTE: int64 cast is needed on osx
-	}, nil
+	return SizeInfo{size}, nil
 }

extern/sector-storage/mock/mock.go

@@ -347,7 +347,7 @@ func generateFakePoSt(sectorInfo []proof2.SectorInfo, rpt func(abi.RegisteredSea
 }
 
 func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error {
-	if len(mgr.sectors[sectorID.ID].pieces) > 1 || offset != 0 {
+	if offset != 0 {
 		panic("implme")
 	}
 

extern/sector-storage/sched_worker.go

@@ -57,6 +57,7 @@ func (sh *scheduler) runWorker(ctx context.Context, w Worker) error {
 		log.Warnw("duplicated worker added", "id", wid)
 
 		// this is ok, we're already handling this worker in a different goroutine
+		sh.workersLk.Unlock()
 		return nil
 	}
 

extern/sector-storage/stores/index.go

@@ -155,6 +155,10 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsS
 			i.stores[si.ID].info.URLs = append(i.stores[si.ID].info.URLs, u)
 		}
 
+		i.stores[si.ID].info.Weight = si.Weight
+		i.stores[si.ID].info.CanSeal = si.CanSeal
+		i.stores[si.ID].info.CanStore = si.CanStore
+
 		return nil
 	}
 	i.stores[si.ID] = &storageEntry{

extern/storage-sealing/cbor_gen.go

@@ -475,7 +475,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{183}); err != nil {
+	if _, err := w.Write([]byte{184, 25}); err != nil {
 		return err
 	}
 
@@ -928,6 +928,50 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	// t.TerminateMessage (cid.Cid) (struct)
+	if len("TerminateMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"TerminateMessage\" was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TerminateMessage"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("TerminateMessage")); err != nil {
+		return err
+	}
+
+	if t.TerminateMessage == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCidBuf(scratch, w, *t.TerminateMessage); err != nil {
+			return xerrors.Errorf("failed to write cid field t.TerminateMessage: %w", err)
+		}
+	}
+
+	// t.TerminatedAt (abi.ChainEpoch) (int64)
+	if len("TerminatedAt") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"TerminatedAt\" was too long")
+	}
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TerminatedAt"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("TerminatedAt")); err != nil {
+		return err
+	}
+
+	if t.TerminatedAt >= 0 {
+		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TerminatedAt)); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TerminatedAt-1)); err != nil {
+			return err
+		}
+	}
+
 	// t.LastErr (string) (string)
 	if len("LastErr") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"LastErr\" was too long")
@@ -1441,6 +1485,55 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 
 				t.Return = ReturnState(sval)
 			}
+			// t.TerminateMessage (cid.Cid) (struct)
+		case "TerminateMessage":
+
+			{
+
+				b, err := br.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := br.UnreadByte(); err != nil {
+						return err
+					}
+
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.TerminateMessage: %w", err)
+					}
+
+					t.TerminateMessage = &c
+				}
+
+			}
+			// t.TerminatedAt (abi.ChainEpoch) (int64)
+		case "TerminatedAt":
+			{
+				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				var extraI int64
+				if err != nil {
+					return err
+				}
+				switch maj {
+				case cbg.MajUnsignedInt:
+					extraI = int64(extra)
+					if extraI < 0 {
+						return fmt.Errorf("int64 positive overflow")
+					}
+				case cbg.MajNegativeInt:
+					extraI = int64(extra)
+					if extraI < 0 {
+						return fmt.Errorf("int64 negative oveflow")
+					}
+					extraI = -1 - extraI
+				default:
+					return fmt.Errorf("wrong type for int64 field: %d", maj)
+				}
+
+				t.TerminatedAt = abi.ChainEpoch(extraI)
+			}
 			// t.LastErr (string) (string)
 		case "LastErr":
 

extern/storage-sealing/fsm.go

@@ -148,6 +148,21 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 		on(SectorFaultReported{}, FaultReported),
 		on(SectorFaulty{}, Faulty),
 	),
+	Terminating: planOne(
+		on(SectorTerminating{}, TerminateWait),
+		on(SectorTerminateFailed{}, TerminateFailed),
+	),
+	TerminateWait: planOne(
+		on(SectorTerminated{}, TerminateFinality),
+		on(SectorTerminateFailed{}, TerminateFailed),
+	),
+	TerminateFinality: planOne(
+		on(SectorTerminateFailed{}, TerminateFailed),
+		// SectorRemove (global)
+	),
+	TerminateFailed: planOne(
+	// SectorTerminating (global)
+	),
 	Removing: planOne(
 		on(SectorRemoved{}, Removed),
 		on(SectorRemoveFailed{}, RemoveFailed),
@@ -328,6 +343,14 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 	// Post-seal
 	case Proving:
 		return m.handleProvingSector, processed, nil
+	case Terminating:
+		return m.handleTerminating, processed, nil
+	case TerminateWait:
+		return m.handleTerminateWait, processed, nil
+	case TerminateFinality:
+		return m.handleTerminateFinality, processed, nil
+	case TerminateFailed:
+		return m.handleTerminateFailed, processed, nil
 	case Removing:
 		return m.handleRemoving, processed, nil
 	case Removed:
@@ -409,8 +432,9 @@ func (m *Sealing) restartSectors(ctx context.Context) error {
 		return err
 	}
 
-	m.unsealedInfoMap.lk.Lock()
+	// m.unsealedInfoMap.lk.Lock() taken early in .New to prevent races
 	defer m.unsealedInfoMap.lk.Unlock()
+
 	for _, sector := range trackedSectors {
 		if err := m.sectors.Send(uint64(sector.SectorNumber), SectorRestart{}); err != nil {
 			log.Errorf("restarting sector %d: %+v", sector.SectorNumber, err)

extern/storage-sealing/fsm_events.go

@@ -314,6 +314,32 @@ func (evt SectorFaultReported) apply(state *SectorInfo) {
 
 type SectorFaultedFinal struct{}
 
+// Terminating
+
+type SectorTerminate struct{}
+
+func (evt SectorTerminate) applyGlobal(state *SectorInfo) bool {
+	state.State = Terminating
+	return true
+}
+
+type SectorTerminating struct{ Message *cid.Cid }
+
+func (evt SectorTerminating) apply(state *SectorInfo) {
+	state.TerminateMessage = evt.Message
+}
+
+type SectorTerminated struct{ TerminatedAt abi.ChainEpoch }
+
+func (evt SectorTerminated) apply(state *SectorInfo) {
+	state.TerminatedAt = evt.TerminatedAt
+}
+
+type SectorTerminateFailed struct{ error }
+
+func (evt SectorTerminateFailed) FormatError(xerrors.Printer) (next error) { return evt.error }
+func (evt SectorTerminateFailed) apply(*SectorInfo)                        {}
+
 // External events
 
 type SectorRemove struct{}

extern/storage-sealing/sealing.go

@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/filecoin-project/go-state-types/network"
 	statemachine "github.com/filecoin-project/go-statemachine"
 	"github.com/filecoin-project/specs-storage/storage"
@@ -60,6 +61,8 @@ type SealingAPI interface {
 	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, TipSetToken) (bool, error)
 	StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error)
 	StateNetworkVersion(ctx context.Context, tok TipSetToken) (network.Version, error)
+	StateMinerProvingDeadline(context.Context, address.Address, TipSetToken) (*dline.Info, error)
+	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tok TipSetToken) ([]api.Partition, error)
 	SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error)
 	ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error)
 	ChainGetRandomnessFromBeacon(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error)
@@ -94,12 +97,15 @@ type Sealing struct {
 
 	stats SectorStats
 
+	terminator *TerminateBatcher
+
 	getConfig GetSealingConfigFunc
 }
 
 type FeeConfig struct {
 	MaxPreCommitGasFee abi.TokenAmount
 	MaxCommitGasFee    abi.TokenAmount
+	MaxTerminateGasFee abi.TokenAmount
 }
 
 type UnsealedSectorMap struct {
@@ -136,6 +142,8 @@ func New(api SealingAPI, fc FeeConfig, events Events, maddr address.Address, ds
 		notifee: notifee,
 		addrSel: as,
 
+		terminator: NewTerminationBatcher(context.TODO(), maddr, api, as, fc),
+
 		getConfig: gc,
 
 		stats: SectorStats{
@@ -145,6 +153,8 @@ func New(api SealingAPI, fc FeeConfig, events Events, maddr address.Address, ds
 
 	s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{})
 
+	s.unsealedInfoMap.lk.Lock() // released after initialized in .Run()
+
 	return s
 }
 
@@ -158,7 +168,14 @@ func (m *Sealing) Run(ctx context.Context) error {
 }
 
 func (m *Sealing) Stop(ctx context.Context) error {
-	return m.sectors.Stop(ctx)
+	if err := m.terminator.Stop(ctx); err != nil {
+		return err
+	}
+
+	if err := m.sectors.Stop(ctx); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) {
@@ -263,6 +280,18 @@ func (m *Sealing) Remove(ctx context.Context, sid abi.SectorNumber) error {
 	return m.sectors.Send(uint64(sid), SectorRemove{})
 }
 
+func (m *Sealing) Terminate(ctx context.Context, sid abi.SectorNumber) error {
+	return m.sectors.Send(uint64(sid), SectorTerminate{})
+}
+
+func (m *Sealing) TerminateFlush(ctx context.Context) (*cid.Cid, error) {
+	return m.terminator.Flush(ctx)
+}
+
+func (m *Sealing) TerminatePending(ctx context.Context) ([]abi.SectorID, error) {
+	return m.terminator.Pending(ctx)
+}
+
 // Caller should NOT hold m.unsealedInfoMap.lk
 func (m *Sealing) StartPacking(sectorID abi.SectorNumber) error {
 	// locking here ensures that when the SectorStartPacking event is sent, the sector won't be picked up anywhere else

extern/storage-sealing/sector_state.go

@@ -30,6 +30,10 @@ var ExistSectorStateList = map[SectorState]struct{}{
 	Faulty:               {},
 	FaultReported:        {},
 	FaultedFinal:         {},
+	Terminating:          {},
+	TerminateWait:        {},
+	TerminateFinality:    {},
+	TerminateFailed:      {},
 	Removing:             {},
 	RemoveFailed:         {},
 	Removed:              {},
@@ -69,6 +73,11 @@ const (
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain
 	FaultedFinal  SectorState = "FaultedFinal"  // fault declared on chain
 
+	Terminating       SectorState = "Terminating"
+	TerminateWait     SectorState = "TerminateWait"
+	TerminateFinality SectorState = "TerminateFinality"
+	TerminateFailed   SectorState = "TerminateFailed"
+
 	Removing     SectorState = "Removing"
 	RemoveFailed SectorState = "RemoveFailed"
 	Removed      SectorState = "Removed"
@@ -78,7 +87,7 @@ func toStatState(st SectorState) statSectorState {
 	switch st {
 	case Empty, WaitDeals, Packing, GetTicket, PreCommit1, PreCommit2, PreCommitting, PreCommitWait, WaitSeed, Committing, SubmitCommit, CommitWait, FinalizeSector:
 		return sstSealing
-	case Proving, Removed, Removing:
+	case Proving, Removed, Removing, Terminating, TerminateWait, TerminateFinality, TerminateFailed:
 		return sstProving
 	}
 

extern/storage-sealing/states_failed.go

@@ -224,9 +224,9 @@ func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo
 		case *ErrBadCommD:
 			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)})
 		case *ErrExpiredTicket:
-			return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)})
+			return ctx.Send(SectorTicketExpired{xerrors.Errorf("ticket expired error, removing sector: %w", err)})
 		case *ErrBadTicket:
-			return ctx.Send(SectorTicketExpired{xerrors.Errorf("expired ticket: %w", err)})
+			return ctx.Send(SectorTicketExpired{xerrors.Errorf("expired ticket, removing sector: %w", err)})
 		case *ErrInvalidDeals:
 			log.Warnf("invalid deals in sector %d: %v", sector.SectorNumber, err)
 			return ctx.Send(SectorInvalidDealIDs{Return: RetCommitFailed})
@@ -309,6 +309,22 @@ func (m *Sealing) handleRemoveFailed(ctx statemachine.Context, sector SectorInfo
 	return ctx.Send(SectorRemove{})
 }
 
+func (m *Sealing) handleTerminateFailed(ctx statemachine.Context, sector SectorInfo) error {
+	// ignoring error as it's most likely an API error - `pci` will be nil, and we'll go back to
+	// the Terminating state after cooldown. If the API is still failing, well get back to here
+	// with the error in SectorInfo log.
+	pci, _ := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)
+	if pci != nil {
+		return nil // pause the fsm, needs manual user action
+	}
+
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorTerminate{})
+}
+
 func (m *Sealing) handleDealsExpired(ctx statemachine.Context, sector SectorInfo) error {
 	// First make vary sure the sector isn't committed
 	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)

extern/storage-sealing/states_proving.go

@@ -1,9 +1,14 @@
 package sealing
 
 import (
+	"time"
+
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/go-statemachine"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 )
 
 func (m *Sealing) handleFaulty(ctx statemachine.Context, sector SectorInfo) error {
@@ -31,6 +36,89 @@ func (m *Sealing) handleFaultReported(ctx statemachine.Context, sector SectorInf
 	return ctx.Send(SectorFaultedFinal{})
 }
 
+func (m *Sealing) handleTerminating(ctx statemachine.Context, sector SectorInfo) error {
+	// First step of sector termination
+	// * See if sector is live
+	//  * If not, goto removing
+	// * Add to termination queue
+	// * Wait for message to land on-chain
+	// * Check for correct termination
+	// * wait for expiration (+winning lookback?)
+
+	si, err := m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)
+	if err != nil {
+		return ctx.Send(SectorTerminateFailed{xerrors.Errorf("getting sector info: %w", err)})
+	}
+
+	if si == nil {
+		// either already terminated or not committed yet
+
+		pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, nil)
+		if err != nil {
+			return ctx.Send(SectorTerminateFailed{xerrors.Errorf("checking precommit presence: %w", err)})
+		}
+		if pci != nil {
+			return ctx.Send(SectorTerminateFailed{xerrors.Errorf("sector was precommitted but not proven, remove instead of terminating")})
+		}
+
+		return ctx.Send(SectorRemove{})
+	}
+
+	termCid, terminated, err := m.terminator.AddTermination(ctx.Context(), m.minerSectorID(sector.SectorNumber))
+	if err != nil {
+		return ctx.Send(SectorTerminateFailed{xerrors.Errorf("queueing termination: %w", err)})
+	}
+
+	if terminated {
+		return ctx.Send(SectorTerminating{Message: nil})
+	}
+
+	return ctx.Send(SectorTerminating{Message: &termCid})
+}
+
+func (m *Sealing) handleTerminateWait(ctx statemachine.Context, sector SectorInfo) error {
+	if sector.TerminateMessage == nil {
+		return xerrors.New("entered TerminateWait with nil TerminateMessage")
+	}
+
+	mw, err := m.api.StateWaitMsg(ctx.Context(), *sector.TerminateMessage)
+	if err != nil {
+		return ctx.Send(SectorTerminateFailed{xerrors.Errorf("waiting for terminate message to land on chain: %w", err)})
+	}
+
+	if mw.Receipt.ExitCode != exitcode.Ok {
+		return ctx.Send(SectorTerminateFailed{xerrors.Errorf("terminate message failed to execute: exit %d: %w", mw.Receipt.ExitCode, err)})
+	}
+
+	return ctx.Send(SectorTerminated{TerminatedAt: mw.Height})
+}
+
+func (m *Sealing) handleTerminateFinality(ctx statemachine.Context, sector SectorInfo) error {
+	for {
+		tok, epoch, err := m.api.ChainHead(ctx.Context())
+		if err != nil {
+			return ctx.Send(SectorTerminateFailed{xerrors.Errorf("getting chain head: %w", err)})
+		}
+
+		nv, err := m.api.StateNetworkVersion(ctx.Context(), tok)
+		if err != nil {
+			return ctx.Send(SectorTerminateFailed{xerrors.Errorf("getting network version: %w", err)})
+		}
+
+		if epoch >= sector.TerminatedAt+policy.GetWinningPoStSectorSetLookback(nv) {
+			return ctx.Send(SectorRemove{})
+		}
+
+		toWait := time.Duration(epoch-sector.TerminatedAt+policy.GetWinningPoStSectorSetLookback(nv)) * time.Duration(build.BlockDelaySecs) * time.Second
+		select {
+		case <-time.After(toWait):
+			continue
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		}
+	}
+}
+
 func (m *Sealing) handleRemoving(ctx statemachine.Context, sector SectorInfo) error {
 	if err := m.sealer.Remove(ctx.Context(), m.minerSector(sector.SectorType, sector.SectorNumber)); err != nil {
 		return ctx.Send(SectorRemoveFailed{err})

extern/storage-sealing/terminate_batch.go

@@ -0,0 +1,351 @@
+package sealing
+
+import (
+	"bytes"
+	"context"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/dline"
+	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+)
+
+var (
+	// TODO: config
+
+	TerminateBatchMax  uint64 = 100 // adjust based on real-world gas numbers, actors limit at 10k
+	TerminateBatchMin  uint64 = 1
+	TerminateBatchWait        = 5 * time.Minute
+)
+
+type TerminateBatcherApi interface {
+	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*SectorLocation, error)
+	SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error)
+	StateMinerInfo(context.Context, address.Address, TipSetToken) (miner.MinerInfo, error)
+	StateMinerProvingDeadline(context.Context, address.Address, TipSetToken) (*dline.Info, error)
+	StateMinerPartitions(ctx context.Context, m address.Address, dlIdx uint64, tok TipSetToken) ([]api.Partition, error)
+}
+
+type TerminateBatcher struct {
+	api     TerminateBatcherApi
+	maddr   address.Address
+	mctx    context.Context
+	addrSel AddrSel
+	feeCfg  FeeConfig
+
+	todo map[SectorLocation]*bitfield.BitField // MinerSectorLocation -> BitField
+
+	waiting map[abi.SectorNumber][]chan cid.Cid
+
+	notify, stop, stopped chan struct{}
+	force                 chan chan *cid.Cid
+	lk                    sync.Mutex
+}
+
+func NewTerminationBatcher(mctx context.Context, maddr address.Address, api TerminateBatcherApi, addrSel AddrSel, feeCfg FeeConfig) *TerminateBatcher {
+	b := &TerminateBatcher{
+		api:     api,
+		maddr:   maddr,
+		mctx:    mctx,
+		addrSel: addrSel,
+		feeCfg:  feeCfg,
+
+		todo:    map[SectorLocation]*bitfield.BitField{},
+		waiting: map[abi.SectorNumber][]chan cid.Cid{},
+
+		notify:  make(chan struct{}, 1),
+		force:   make(chan chan *cid.Cid),
+		stop:    make(chan struct{}),
+		stopped: make(chan struct{}),
+	}
+
+	go b.run()
+
+	return b
+}
+
+func (b *TerminateBatcher) run() {
+	var forceRes chan *cid.Cid
+	var lastMsg *cid.Cid
+
+	for {
+		if forceRes != nil {
+			forceRes <- lastMsg
+			forceRes = nil
+		}
+		lastMsg = nil
+
+		var sendAboveMax, sendAboveMin bool
+		select {
+		case <-b.stop:
+			close(b.stopped)
+			return
+		case <-b.notify:
+			sendAboveMax = true
+		case <-time.After(TerminateBatchWait):
+			sendAboveMin = true
+		case fr := <-b.force: // user triggered
+			forceRes = fr
+		}
+
+		var err error
+		lastMsg, err = b.processBatch(sendAboveMax, sendAboveMin)
+		if err != nil {
+			log.Warnw("TerminateBatcher processBatch error", "error", err)
+		}
+	}
+}
+
+func (b *TerminateBatcher) processBatch(notif, after bool) (*cid.Cid, error) {
+	dl, err := b.api.StateMinerProvingDeadline(b.mctx, b.maddr, nil)
+	if err != nil {
+		return nil, xerrors.Errorf("getting proving deadline info failed: %w", err)
+	}
+
+	b.lk.Lock()
+	defer b.lk.Unlock()
+	params := miner2.TerminateSectorsParams{}
+
+	var total uint64
+	for loc, sectors := range b.todo {
+		n, err := sectors.Count()
+		if err != nil {
+			log.Errorw("TerminateBatcher: failed to count sectors to terminate", "deadline", loc.Deadline, "partition", loc.Partition, "error", err)
+			continue
+		}
+
+		// don't send terminations for currently challenged sectors
+		if loc.Deadline == (dl.Index+1)%miner.WPoStPeriodDeadlines || // not in next (in case the terminate message takes a while to get on chain)
+			loc.Deadline == dl.Index || // not in current
+			(loc.Deadline+1)%miner.WPoStPeriodDeadlines == dl.Index { // not in previous
+			continue
+		}
+
+		if n < 1 {
+			log.Warnw("TerminateBatcher: zero sectors in bucket", "deadline", loc.Deadline, "partition", loc.Partition)
+			continue
+		}
+
+		toTerminate, err := sectors.Copy()
+		if err != nil {
+			log.Warnw("TerminateBatcher: copy sectors bitfield", "deadline", loc.Deadline, "partition", loc.Partition, "error", err)
+			continue
+		}
+
+		if total+n > uint64(miner.AddressedSectorsMax) {
+			n = uint64(miner.AddressedSectorsMax) - total
+
+			toTerminate, err = toTerminate.Slice(0, n)
+			if err != nil {
+				log.Warnw("TerminateBatcher: slice toTerminate bitfield", "deadline", loc.Deadline, "partition", loc.Partition, "error", err)
+				continue
+			}
+
+			s, err := bitfield.SubtractBitField(*sectors, toTerminate)
+			if err != nil {
+				log.Warnw("TerminateBatcher: sectors-toTerminate", "deadline", loc.Deadline, "partition", loc.Partition, "error", err)
+				continue
+			}
+			*sectors = s
+		}
+
+		total += n
+
+		params.Terminations = append(params.Terminations, miner2.TerminationDeclaration{
+			Deadline:  loc.Deadline,
+			Partition: loc.Partition,
+			Sectors:   toTerminate,
+		})
+
+		if total >= uint64(miner.AddressedSectorsMax) {
+			break
+		}
+
+		if len(params.Terminations) >= miner.DeclarationsMax {
+			break
+		}
+	}
+
+	if len(params.Terminations) == 0 {
+		return nil, nil // nothing to do
+	}
+
+	if notif && total < TerminateBatchMax {
+		return nil, nil
+	}
+
+	if after && total < TerminateBatchMin {
+		return nil, nil
+	}
+
+	enc := new(bytes.Buffer)
+	if err := params.MarshalCBOR(enc); err != nil {
+		return nil, xerrors.Errorf("couldn't serialize TerminateSectors params: %w", err)
+	}
+
+	mi, err := b.api.StateMinerInfo(b.mctx, b.maddr, nil)
+	if err != nil {
+		return nil, xerrors.Errorf("couldn't get miner info: %w", err)
+	}
+
+	from, _, err := b.addrSel(b.mctx, mi, api.TerminateSectorsAddr, b.feeCfg.MaxTerminateGasFee, b.feeCfg.MaxTerminateGasFee)
+	if err != nil {
+		return nil, xerrors.Errorf("no good address found: %w", err)
+	}
+
+	mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.TerminateSectors, big.Zero(), b.feeCfg.MaxTerminateGasFee, enc.Bytes())
+	if err != nil {
+		return nil, xerrors.Errorf("sending message failed: %w", err)
+	}
+	log.Infow("Sent TerminateSectors message", "cid", mcid, "from", from, "terminations", len(params.Terminations))
+
+	for _, t := range params.Terminations {
+		delete(b.todo, SectorLocation{
+			Deadline:  t.Deadline,
+			Partition: t.Partition,
+		})
+
+		err := t.Sectors.ForEach(func(sn uint64) error {
+			for _, ch := range b.waiting[abi.SectorNumber(sn)] {
+				ch <- mcid // buffered
+			}
+			delete(b.waiting, abi.SectorNumber(sn))
+
+			return nil
+		})
+		if err != nil {
+			return nil, xerrors.Errorf("sectors foreach: %w", err)
+		}
+	}
+
+	return &mcid, nil
+}
+
+// register termination, wait for batch message, return message CID
+// can return cid.Undef,true if the sector is already terminated on-chain
+func (b *TerminateBatcher) AddTermination(ctx context.Context, s abi.SectorID) (mcid cid.Cid, terminated bool, err error) {
+	maddr, err := address.NewIDAddress(uint64(s.Miner))
+	if err != nil {
+		return cid.Undef, false, err
+	}
+
+	loc, err := b.api.StateSectorPartition(ctx, maddr, s.Number, nil)
+	if err != nil {
+		return cid.Undef, false, xerrors.Errorf("getting sector location: %w", err)
+	}
+	if loc == nil {
+		return cid.Undef, false, xerrors.New("sector location not found")
+	}
+
+	{
+		// check if maybe already terminated
+		parts, err := b.api.StateMinerPartitions(ctx, maddr, loc.Deadline, nil)
+		if err != nil {
+			return cid.Cid{}, false, xerrors.Errorf("getting partitions: %w", err)
+		}
+		live, err := parts[loc.Partition].LiveSectors.IsSet(uint64(s.Number))
+		if err != nil {
+			return cid.Cid{}, false, xerrors.Errorf("checking if sector is in live set: %w", err)
+		}
+		if !live {
+			// already terminated
+			return cid.Undef, true, nil
+		}
+	}
+
+	b.lk.Lock()
+	bf, ok := b.todo[*loc]
+	if !ok {
+		n := bitfield.New()
+		bf = &n
+		b.todo[*loc] = bf
+	}
+	bf.Set(uint64(s.Number))
+
+	sent := make(chan cid.Cid, 1)
+	b.waiting[s.Number] = append(b.waiting[s.Number], sent)
+
+	select {
+	case b.notify <- struct{}{}:
+	default: // already have a pending notification, don't need more
+	}
+	b.lk.Unlock()
+
+	select {
+	case c := <-sent:
+		return c, false, nil
+	case <-ctx.Done():
+		return cid.Undef, false, ctx.Err()
+	}
+}
+
+func (b *TerminateBatcher) Flush(ctx context.Context) (*cid.Cid, error) {
+	resCh := make(chan *cid.Cid, 1)
+	select {
+	case b.force <- resCh:
+		select {
+		case res := <-resCh:
+			return res, nil
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func (b *TerminateBatcher) Pending(ctx context.Context) ([]abi.SectorID, error) {
+	b.lk.Lock()
+	defer b.lk.Unlock()
+
+	mid, err := address.IDFromAddress(b.maddr)
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]abi.SectorID, 0)
+	for _, bf := range b.todo {
+		err := bf.ForEach(func(id uint64) error {
+			res = append(res, abi.SectorID{
+				Miner:  abi.ActorID(mid),
+				Number: abi.SectorNumber(id),
+			})
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	sort.Slice(res, func(i, j int) bool {
+		if res[i].Miner != res[j].Miner {
+			return res[i].Miner < res[j].Miner
+		}
+
+		return res[i].Number < res[j].Number
+	})
+
+	return res, nil
+}
+
+func (b *TerminateBatcher) Stop(ctx context.Context) error {
+	close(b.stop)
+
+	select {
+	case <-b.stopped:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

extern/storage-sealing/types.go

@@ -103,6 +103,10 @@ type SectorInfo struct {
 	// Recovery
 	Return ReturnState
 
+	// Termination
+	TerminateMessage *cid.Cid
+	TerminatedAt     abi.ChainEpoch
+
 	// Debug
 	LastErr string
 

go.mod

@@ -104,7 +104,7 @@ require (
 	github.com/libp2p/go-libp2p-mplex v0.3.0
 	github.com/libp2p/go-libp2p-noise v0.1.2
 	github.com/libp2p/go-libp2p-peerstore v0.2.6
-	github.com/libp2p/go-libp2p-pubsub v0.4.0
+	github.com/libp2p/go-libp2p-pubsub v0.4.1
 	github.com/libp2p/go-libp2p-quic-transport v0.9.0
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3

go.sum

@@ -925,8 +925,8 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1
 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk=
 github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q=
 github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato=
-github.com/libp2p/go-libp2p-pubsub v0.4.0 h1:YNVRyXqBgv9i4RG88jzoTtkSOaSB45CqHkL29NNBZb4=
-github.com/libp2p/go-libp2p-pubsub v0.4.0/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ=
+github.com/libp2p/go-libp2p-pubsub v0.4.1 h1:j4umIg5nyus+sqNfU+FWvb9aeYFQH/A+nDFhWj+8yy8=
+github.com/libp2p/go-libp2p-pubsub v0.4.1/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ=
 github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU=
 github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M=
 github.com/libp2p/go-libp2p-quic-transport v0.9.0 h1:WPuq5nV/chmIZIzvrkC2ulSdAQ0P0BDvgvAhZFOZ59E=

lib/blockstore/badger/blockstore_test_suite.go

@@ -183,6 +183,8 @@ func (s *Suite) TestAllKeysRespectsContext(t *testing.T) {
 	require.True(t, ok)
 
 	cancel()
+	// pull one value out to avoid race
+	_, _ = <-ch
 
 	v, ok = <-ch
 	require.Equal(t, cid.Undef, v)

node/config/def.go

@@ -69,6 +69,7 @@ type SealingConfig struct {
 type MinerFeeConfig struct {
 	MaxPreCommitGasFee     types.FIL
 	MaxCommitGasFee        types.FIL
+	MaxTerminateGasFee     types.FIL
 	MaxWindowPoStGasFee    types.FIL
 	MaxPublishDealsFee     types.FIL
 	MaxMarketBalanceAddFee types.FIL
@@ -211,6 +212,7 @@ func DefaultStorageMiner() *StorageMiner {
 		Fees: MinerFeeConfig{
 			MaxPreCommitGasFee:     types.MustParseFIL("0.025"),
 			MaxCommitGasFee:        types.MustParseFIL("0.05"),
+			MaxTerminateGasFee:     types.MustParseFIL("0.5"),
 			MaxWindowPoStGasFee:    types.MustParseFIL("5"),
 			MaxPublishDealsFee:     types.MustParseFIL("0.05"),
 			MaxMarketBalanceAddFee: types.MustParseFIL("0.007"),

node/impl/full/gas.go

@@ -93,13 +93,15 @@ type gasMeta struct {
 	limit int64
 }
 
+// finds 55th percntile instead of median to put negative pressure on gas price
 func medianGasPremium(prices []gasMeta, blocks int) abi.TokenAmount {
 	sort.Slice(prices, func(i, j int) bool {
 		// sort desc by price
 		return prices[i].price.GreaterThan(prices[j].price)
 	})
 
-	at := build.BlockGasTarget * int64(blocks) / 2
+	at := build.BlockGasTarget * int64(blocks) / 2        // 50th
+	at += build.BlockGasTarget * int64(blocks) / (2 * 20) // move 5% further
 	prev1, prev2 := big.Zero(), big.Zero()
 	for _, price := range prices {
 		prev1, prev2 = price.price, prev1

node/impl/full/state.go

@@ -55,6 +55,7 @@ type StateModuleAPI interface {
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
 	StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
+	StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
 	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error)
 }
@@ -589,8 +590,14 @@ func stateWaitMsgLimited(ctx context.Context, smgr *stmgr.StateManager, cstore *
 	}, nil
 }
 
-func (a *StateAPI) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
-	ts, recpt, found, err := a.StateManager.SearchForMessage(ctx, msg)
+func (m *StateModule) StateSearchMsg(ctx context.Context, msg cid.Cid) (*api.MsgLookup, error) {
+	return stateSearchMsgLimited(ctx, m.StateManager, msg, stmgr.LookbackNoLimit)
+}
+func (a *StateAPI) StateSearchMsgLimited(ctx context.Context, msg cid.Cid, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) {
+	return stateSearchMsgLimited(ctx, a.StateManager, msg, lookbackLimit)
+}
+func stateSearchMsgLimited(ctx context.Context, smgr *stmgr.StateManager, msg cid.Cid, lookbackLimit abi.ChainEpoch) (*api.MsgLookup, error) {
+	ts, recpt, found, err := smgr.SearchForMessage(ctx, msg, lookbackLimit)
 	if err != nil {
 		return nil, err
 	}

node/impl/market/market.go

@@ -3,21 +3,49 @@ package market
 import (
 	"context"
 
+	"github.com/ipfs/go-cid"
 	"go.uber.org/fx"
 
-	"github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/actors"
+	marketactor "github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/market"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/impl/full"
 )
 
 type MarketAPI struct {
 	fx.In
 
+	full.MpoolAPI
 	FMgr *market.FundManager
 }
 
+func (a *MarketAPI) MarketAddBalance(ctx context.Context, wallet, addr address.Address, amt types.BigInt) (cid.Cid, error) {
+	params, err := actors.SerializeParams(&addr)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	smsg, aerr := a.MpoolPushMessage(ctx, &types.Message{
+		To:     marketactor.Address,
+		From:   wallet,
+		Value:  amt,
+		Method: marketactor.Methods.AddBalance,
+		Params: params,
+	}, nil)
+
+	if aerr != nil {
+		return cid.Undef, aerr
+	}
+
+	return smsg.Cid(), nil
+}
+
+func (a *MarketAPI) MarketGetReserved(ctx context.Context, addr address.Address) (types.BigInt, error) {
+	return a.FMgr.GetReserved(addr), nil
+}
+
 func (a *MarketAPI) MarketReserveFunds(ctx context.Context, wallet address.Address, addr address.Address, amt types.BigInt) (cid.Cid, error) {
 	return a.FMgr.Reserve(ctx, wallet, addr, amt)
 }

node/impl/storminer.go

@@ -328,6 +328,18 @@ func (sm *StorageMinerAPI) SectorRemove(ctx context.Context, id abi.SectorNumber
 	return sm.Miner.RemoveSector(ctx, id)
 }
 
+func (sm *StorageMinerAPI) SectorTerminate(ctx context.Context, id abi.SectorNumber) error {
+	return sm.Miner.TerminateSector(ctx, id)
+}
+
+func (sm *StorageMinerAPI) SectorTerminateFlush(ctx context.Context) (*cid.Cid, error) {
+	return sm.Miner.TerminateFlush(ctx)
+}
+
+func (sm *StorageMinerAPI) SectorTerminatePending(ctx context.Context) ([]abi.SectorID, error) {
+	return sm.Miner.TerminatePending(ctx)
+}
+
 func (sm *StorageMinerAPI) SectorMarkForUpgrade(ctx context.Context, id abi.SectorNumber) error {
 	return sm.Miner.MarkForUpgrade(id)
 }

node/node_test.go

@@ -164,6 +164,20 @@ func TestWindowedPost(t *testing.T) {
 	test.TestWindowPost(t, builder.MockSbBuilder, 2*time.Millisecond, 10)
 }
 
+func TestTerminate(t *testing.T) {
+	if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" {
+		t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run")
+	}
+
+	logging.SetLogLevel("miner", "ERROR")
+	logging.SetLogLevel("chainstore", "ERROR")
+	logging.SetLogLevel("chain", "ERROR")
+	logging.SetLogLevel("sub", "ERROR")
+	logging.SetLogLevel("storageminer", "ERROR")
+
+	test.TestTerminate(t, builder.MockSbBuilder, 2*time.Millisecond)
+}
+
 func TestCCUpgrade(t *testing.T) {
 	logging.SetLogLevel("miner", "ERROR")
 	logging.SetLogLevel("chainstore", "ERROR")

storage/adapter_storage_miner.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"context"
 
-	"github.com/filecoin-project/go-state-types/network"
-
 	"github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
@@ -14,6 +12,8 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/dline"
+	"github.com/filecoin-project/go-state-types/network"
 
 	market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
 
@@ -243,6 +243,15 @@ func (s SealingAPIAdapter) StateSectorPartition(ctx context.Context, maddr addre
 	return nil, nil // not found
 }
 
+func (s SealingAPIAdapter) StateMinerPartitions(ctx context.Context, maddr address.Address, dlIdx uint64, tok sealing.TipSetToken) ([]api.Partition, error) {
+	tsk, err := types.TipSetKeyFromBytes(tok)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err)
+	}
+
+	return s.delegate.StateMinerPartitions(ctx, maddr, dlIdx, tsk)
+}
+
 func (s SealingAPIAdapter) StateMarketStorageDeal(ctx context.Context, dealID abi.DealID, tok sealing.TipSetToken) (market.DealProposal, error) {
 	tsk, err := types.TipSetKeyFromBytes(tok)
 	if err != nil {
@@ -266,6 +275,15 @@ func (s SealingAPIAdapter) StateNetworkVersion(ctx context.Context, tok sealing.
 	return s.delegate.StateNetworkVersion(ctx, tsk)
 }
 
+func (s SealingAPIAdapter) StateMinerProvingDeadline(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (*dline.Info, error) {
+	tsk, err := types.TipSetKeyFromBytes(tok)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.delegate.StateMinerProvingDeadline(ctx, maddr, tsk)
+}
+
 func (s SealingAPIAdapter) SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error) {
 	msg := types.Message{
 		To:     to,

storage/addresses.go

@@ -30,6 +30,8 @@ func (as *AddressSelector) AddressFor(ctx context.Context, a addrSelectApi, mi m
 		addrs = append(addrs, as.PreCommitControl...)
 	case api.CommitAddr:
 		addrs = append(addrs, as.CommitControl...)
+	case api.TerminateSectorsAddr:
+		addrs = append(addrs, as.TerminateControl...)
 	default:
 		defaultCtl := map[address.Address]struct{}{}
 		for _, a := range mi.ControlAddresses {
@@ -129,6 +131,6 @@ func maybeUseAddress(ctx context.Context, a addrSelectApi, addr address.Address,
 		*bestAvail = b
 	}
 
-	log.Warnw("address didn't have enough funds for window post message", "address", addr, "required", types.FIL(goodFunds), "balance", types.FIL(b))
+	log.Warnw("address didn't have enough funds to send message", "address", addr, "required", types.FIL(goodFunds), "balance", types.FIL(b))
 	return false
 }

storage/miner.go

@@ -148,6 +148,7 @@ func (m *Miner) Run(ctx context.Context) error {
 	fc := sealing.FeeConfig{
 		MaxPreCommitGasFee: abi.TokenAmount(m.feeCfg.MaxPreCommitGasFee),
 		MaxCommitGasFee:    abi.TokenAmount(m.feeCfg.MaxCommitGasFee),
+		MaxTerminateGasFee: abi.TokenAmount(m.feeCfg.MaxTerminateGasFee),
 	}
 
 	evts := events.NewEvents(ctx, m.api)

storage/sealing.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"io"
 
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 
@@ -44,6 +46,18 @@ func (m *Miner) RemoveSector(ctx context.Context, id abi.SectorNumber) error {
 	return m.sealing.Remove(ctx, id)
 }
 
+func (m *Miner) TerminateSector(ctx context.Context, id abi.SectorNumber) error {
+	return m.sealing.Terminate(ctx, id)
+}
+
+func (m *Miner) TerminateFlush(ctx context.Context) (*cid.Cid, error) {
+	return m.sealing.TerminateFlush(ctx)
+}
+
+func (m *Miner) TerminatePending(ctx context.Context) ([]abi.SectorID, error) {
+	return m.sealing.TerminatePending(ctx)
+}
+
 func (m *Miner) MarkForUpgrade(id abi.SectorNumber) error {
 	return m.sealing.MarkForUpgrade(id)
 }