add index to event.emitter_addr

feat: worker: Support delegating precommit2 to external binary

.circleci/config.yml

@@ -996,23 +996,20 @@ workflows:
             - build
           suite: utest-unit-cli
           target: "./cli/... ./cmd/... ./api/..."
-          get-params: true
           executor: golang-2xl
+          get-params: true
       - test:
           name: test-unit-node
           requires:
             - build
           suite: utest-unit-node
           target: "./node/..."
-          
-          
       - test:
           name: test-unit-rest
           requires:
             - build
           suite: utest-unit-rest
           target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
-          
           executor: golang-2xl
       - test:
           name: test-unit-storage
@@ -1020,8 +1017,7 @@ workflows:
             - build
           suite: utest-unit-storage
           target: "./storage/... ./extern/..."
-          
-          
+          get-params: true
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           requires:

.circleci/template.yml

@@ -557,9 +557,16 @@ workflows:
             - build
           suite: utest-[[ $suite ]]
           target: "[[ $pkgs ]]"
-          [[if eq $suite "unit-cli"]]get-params: true[[end]]
-          [[if eq $suite "unit-cli"]]executor: golang-2xl[[end]]
-          [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]]
+          [[- if eq $suite "unit-storage"]]
+          get-params: true
+          [[- end -]]
+          [[- if eq $suite "unit-cli"]]
+          executor: golang-2xl
+          get-params: true
+          [[- end -]]
+          [[- if eq $suite "unit-rest"]]
+          executor: golang-2xl
+          [[- end -]]
       [[- end]]
       - test:
           go-test-flags: "-run=TestMulticoreSDR"

.gitignore

@@ -41,6 +41,7 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
+.vscode
 scratchpad
 
 build/builtin-actors/v*

.golangci.yml

@@ -14,6 +14,7 @@ linters:
     - varcheck
     - deadcode
     - scopelint
+    - unused
 
 # We don't want to skip builtin/
 skip-dirs-use-default: false

CHANGELOG.md

@@ -1,14 +1,21 @@
 # Lotus changelog
 
-# v1.25.1 / 2023-12-09
-
-This is a **highly recommended PATCH RELEASE.**  The patch release fixes the issue were node operators trying to catch up sync were unable to sync large message blocks/epochs due to an increased number of messages on the network.
-
-This patch release allows for up to 10k messages per block. Additionally, it introduces a limit on the amount of data that can be read at once, ensuring the system can handle worst-case scenarios.
+# UNRELEASED
+- chore: Auto remove local chain data when importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
+- feat: metric: export Mpool message count ([filecoin-project/lotus#11361](https://github.com/filecoin-project/lotus/pull/11361))
+- feat: sealing: load SectorsSummary from sealing SectorStats instead of calling API each time ([filecoin-project/lotus#11353](https://github.com/filecoin-project/lotus/pull/11353))
+- fix: miner info: Show correct sector state counts ([filecoin-project/lotus#11456](https://github.com/filecoin-project/lotus/pull/11456))
+- feat: add support for specifying block number when calling `eth_estimateGas` ([filecoin-project/lotus#11462](https://github.com/filecoin-project/lotus/pull/11462)).
 
 ## Improvements
-- fix: exchange: allow up to 10k messages per block ([filecoin-project/lotus#11506](https://github.com/filecoin-project/lotus/pull/11506))
-
+- fix: Add time slicing to splitstore purging step during compaction to reduce lock congestion [filecoin-project/lotus#11269](https://github.com/filecoin-project/lotus/pull/11269)
+- feat: Added instructions on how to setup Prometheus/Grafana for monitoring a local Lotus node [filecoin-project/lotus#11276](https://github.com/filecoin-project/lotus/pull/11276)
+- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
+- fix: The Ethereum API will now use the correct state-tree when resolving "native" addresses into masked ID addresses. Additionally, pending messages from native account types won't be visible in the Ethereum API because there is no "correct" state-tree to pick in this case. However, pending _Ethereum_ transactions and native messages that have landed on-chain will still be visible through the Ethereum API.
+- feat: Unambiguously translate native messages to Ethereum transactions by:
+  - Detecting native messages that "look" like Ethereum transactions (creating smart contracts, invoking a smart contract, etc.), and decoding them as such.
+  - Otherwise, ABI-encoding the inputs as if they were calls to a `handle_filecoin_method` Solidity method.
+- fix: ensure that the Ethereum API never returns "empty" addresses for native messages. When a "to" address cannot be resolved to a 0x-style address, it will be re-written to `0xff0000000000000000000000ffffffffffffffff`. This can only happen when the native transaction _reverted_ (failing to create an account at the specified "to" address).
 
 # v 1.25.0 / 2023-11-22
 
@@ -94,7 +101,6 @@ Lotus-workers can now be built to leverage the SupraSeal C2 sealing optimization
 - fix(client): single-root error message ([filecoin-project/lotus#11214](https://github.com/filecoin-project/lotus/pull/11214))
 - fix: worker: Convert `DC_[SectorSize]_[ResourceRestriction]` if set ([filecoin-project/lotus#11224](https://github.com/filecoin-project/lotus/pull/11224))
 - chore: backport #11338 onto release/v1.25.0 ([filecoin-project/lotus#11350](https://github.com/filecoin-project/lotus/pull/11350))
-- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
 
 ## Dependencies
 - deps: update go-libp2p to v0.28.1 ([filecoin-project/lotus#10998](https://github.com/filecoin-project/lotus/pull/10998))
@@ -256,7 +262,7 @@ This feature release requires a **minimum Go version of v1.19.12 or higher to su
   - feat: sealing: flag to run data_cid untied from addpiece ([filecoin-project/lotus#10797](https://github.com/filecoin-project/lotus/pull/10797))
   - feat: Lotus Gateway: add MpoolPending, ChainGetBlock and MinerGetBaseInfo ([filecoin-project/lotus#10929](https://github.com/filecoin-project/lotus/pull/10929))
 
-## Improvements && Bug Fixe
+## Improvements && Bug Fixes
   - chore: update ffi & fvm ([filecoin-project/lotus#11040](https://github.com/filecoin-project/lotus/pull/11040))
   - feat: Make sure we don't store duplidate actor events caused to reorgs in events.db ([filecoin-project/lotus#11015](https://github.com/filecoin-project/lotus/pull/11015))
   - sealing: Use only non-assigned deals when selecting snap sectors ([filecoin-project/lotus#11002](https://github.com/filecoin-project/lotus/pull/11002))

LOTUS_RELEASE_FLOW.md

@@ -73,7 +73,7 @@ All releases under an odd minor version number indicate **feature releases**. Th
 
 Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes.  There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
 
-We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the Release Cycle section (TODO: Link).
+We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the [Release Cycle section](#release-cycle).
 
 ### Examples Scenarios
 

README.md

@@ -71,10 +71,10 @@ For other distributions you can find the required dependencies [here.](https://l
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.19.12 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.20.10 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.19.12.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.20.10.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**
@@ -133,6 +133,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
 6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
 
+7. (Optional) Follow the [Setting Up Prometheus and Grafana](https://github.com/filecoin-project/lotus/tree/master/metrics/README.md) guide for detailed instructions on setting up a working monitoring system running against a local running lotus node.
+
 ## License
 
 Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE)

api/api_full.go

@@ -824,7 +824,7 @@ type FullNode interface {
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)                                                                          //perm:read
 
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)                                             //perm:read
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)                                 //perm:read
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)                                 //perm:read
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
 
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read

api/api_gateway.go

@@ -114,7 +114,7 @@ type Gateway interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)

api/mocks/mock_full.go

@@ -1042,7 +1042,7 @@ func (mr *MockFullNodeMockRecorder) EthChainId(arg0 interface{}) *gomock.Call {
 }
 
 // EthEstimateGas mocks base method.
-func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "EthEstimateGas", arg0, arg1)
 	ret0, _ := ret[0].(ethtypes.EthUint64)

api/proxy_gen.go

@@ -255,7 +255,7 @@ type FullNodeMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) `perm:"read"`
 
-	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) `perm:"read"`
+	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) `perm:"read"`
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) `perm:"read"`
 
@@ -679,7 +679,7 @@ type GatewayMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) ``
 
-	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) ``
+	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) ``
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) ``
 
@@ -2134,14 +2134,14 @@ func (s *FullNodeStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
@@ -4400,14 +4400,14 @@ func (s *GatewayStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error)
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 

api/types.go

@@ -56,9 +56,17 @@ type PubsubScore struct {
 	Score *pubsub.PeerScoreSnapshot
 }
 
+// MessageSendSpec contains optional fields which modify message sending behavior
 type MessageSendSpec struct {
-	MaxFee  abi.TokenAmount
+	// MaxFee specifies a cap on network fees related to this message
+	MaxFee abi.TokenAmount
+
+	// MsgUuid specifies a unique message identifier which can be used on node (or node cluster)
+	// level to prevent double-sends of messages even when nonce generation is not handled by sender
 	MsgUuid uuid.UUID
+
+	// MaximizeFeeCap makes message FeeCap be based entirely on MaxFee
+	MaximizeFeeCap bool
 }
 
 type MpoolMessageWhole struct {

blockstore/idstore.go

@@ -183,3 +183,17 @@ func (b *idstore) Close() error {
 func (b *idstore) Flush(ctx context.Context) error {
 	return b.bs.Flush(ctx)
 }
+
+func (b *idstore) CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error {
+	if bs, ok := b.bs.(BlockstoreGC); ok {
+		return bs.CollectGarbage(ctx, options...)
+	}
+	return xerrors.Errorf("not supported")
+}
+
+func (b *idstore) GCOnce(ctx context.Context, options ...BlockstoreGCOption) error {
+	if bs, ok := b.bs.(BlockstoreGCOnce); ok {
+		return bs.GCOnce(ctx, options...)
+	}
+	return xerrors.Errorf("not supported")
+}

blockstore/splitstore/splitstore.go

@@ -182,7 +182,6 @@ type SplitStore struct {
 
 	compactionIndex int64
 	pruneIndex      int64
-	onlineGCCnt     int64
 
 	ctx    context.Context
 	cancel func()

blockstore/splitstore/splitstore_compact.go

@@ -66,8 +66,9 @@ var (
 )
 
 const (
-	batchSize  = 16384
-	cidKeySize = 128
+	batchSize              = 16384
+	cidKeySize             = 128
+	purgeWorkSliceDuration = time.Second
 )
 
 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -1372,9 +1373,21 @@ func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet
 		return err
 	}
 
+	now := time.Now()
+
 	err := coldr.ForEach(func(c cid.Cid) error {
 		batch = append(batch, c)
 		if len(batch) == batchSize {
+			// add some time slicing to the purge as this a very disk I/O heavy operation that
+			// requires write access to txnLk that may starve other operations that require
+			// access to the blockstore.
+			elapsed := time.Since(now)
+			if elapsed > purgeWorkSliceDuration {
+				// work 1 slice, sleep 4 slices, or 20% utilization
+				time.Sleep(4 * elapsed)
+				now = time.Now()
+			}
+
 			return deleteBatch()
 		}
 

build/bootstrap/mainnet.pi

@@ -1,9 +1,6 @@
 /dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj
 /dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc
 /dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4
-/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R
-/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc
-/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH
 /dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ
 /dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf
 /dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR
@@ -11,7 +8,5 @@
 /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz
 /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
-/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
-/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
 /dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
 /dns4/bootstrap-venus.mainnet.filincubator.com/tcp/8888/p2p/QmQu8C6deXwKvJP2D8B6QGyhngc3ZiDnFzEHBDx8yeBXST

build/isnearupgrade.go

@@ -5,5 +5,8 @@ import (
 )
 
 func IsNearUpgrade(epoch, upgradeEpoch abi.ChainEpoch) bool {
+	if upgradeEpoch < 0 {
+		return false
+	}
 	return epoch > upgradeEpoch-Finality && epoch < upgradeEpoch+Finality
 }

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.25.1"
+const BuildVersion = "1.25.1-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/builtin/miner/state.go.template

@@ -72,7 +72,7 @@ func (s *state{{.v}}) AvailableBalance(bal abi.TokenAmount) (available abi.Token
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available{{if (ge .v 2)}}, err{{end}} = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v0.go

@@ -62,7 +62,7 @@ func (s *state0) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v10.go

@@ -62,7 +62,7 @@ func (s *state10) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v11.go

@@ -62,7 +62,7 @@ func (s *state11) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v12.go

@@ -62,7 +62,7 @@ func (s *state12) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v2.go

@@ -61,7 +61,7 @@ func (s *state2) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v3.go

@@ -62,7 +62,7 @@ func (s *state3) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v4.go

@@ -62,7 +62,7 @@ func (s *state4) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v5.go

@@ -62,7 +62,7 @@ func (s *state5) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v6.go

@@ -62,7 +62,7 @@ func (s *state6) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v7.go

@@ -62,7 +62,7 @@ func (s *state7) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v8.go

@@ -62,7 +62,7 @@ func (s *state8) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v9.go

@@ -62,7 +62,7 @@ func (s *state9) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesnt have enough funds to cover their locked pledge
+	// this panics if the miner doesn't have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/paych/actor.go.template

@@ -29,7 +29,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-// Load returns an abstract copy of payment channel state, irregardless of actor version
+// Load returns an abstract copy of payment channel state, regardless of actor version
 func Load(store adt.Store, act *types.Actor) (State, error) {
 	if name, av, ok := actors.GetActorMetaByCode(act.Code); ok {
        if name != manifest.PaychKey {

chain/actors/builtin/paych/paych.go

@@ -29,7 +29,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-// Load returns an abstract copy of payment channel state, irregardless of actor version
+// Load returns an abstract copy of payment channel state, regardless of actor version
 func Load(store adt.Store, act *types.Actor) (State, error) {
 	if name, av, ok := actors.GetActorMetaByCode(act.Code); ok {
 		if name != manifest.PaychKey {

chain/actors/params.go

@@ -13,7 +13,7 @@ import (
 func SerializeParams(i cbg.CBORMarshaler) ([]byte, aerrors.ActorError) {
 	buf := new(bytes.Buffer)
 	if err := i.MarshalCBOR(buf); err != nil {
-		// TODO: shouldnt this be a fatal error?
+		// TODO: shouldn't this be a fatal error?
 		return nil, aerrors.Absorb(err, exitcode.ErrSerialization, "failed to encode parameter")
 	}
 	return buf.Bytes(), nil

chain/beacon/drand/drand.go

@@ -29,19 +29,6 @@ import (
 
 var log = logging.Logger("drand")
 
-type drandPeer struct {
-	addr string
-	tls  bool
-}
-
-func (dp *drandPeer) Address() string {
-	return dp.addr
-}
-
-func (dp *drandPeer) IsTLS() bool {
-	return dp.tls
-}
-
 // DrandBeacon connects Lotus with a drand network in order to provide
 // randomness to the system in a way that's aligned with Filecoin rounds/epochs.
 //

chain/beacon/drand/drand_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func TestPrintGroupInfo(t *testing.T) {
-	server := build.DrandConfigs[build.DrandDevnet].Servers[0]
+	server := build.DrandConfigs[build.DrandTestnet].Servers[0]
 	c, err := hclient.New(server, nil, nil)
 	assert.NoError(t, err)
 	cg := c.(interface {
@@ -31,7 +31,7 @@ func TestPrintGroupInfo(t *testing.T) {
 
 func TestMaxBeaconRoundForEpoch(t *testing.T) {
 	todayTs := uint64(1652222222)
-	db, err := NewDrandBeacon(todayTs, build.BlockDelaySecs, nil, build.DrandConfigs[build.DrandDevnet])
+	db, err := NewDrandBeacon(todayTs, build.BlockDelaySecs, nil, build.DrandConfigs[build.DrandTestnet])
 	assert.NoError(t, err)
 	mbr15 := db.MaxBeaconRoundForEpoch(network.Version15, 100)
 	mbr16 := db.MaxBeaconRoundForEpoch(network.Version16, 100)

chain/consensus/compute_state.go

@@ -147,9 +147,6 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
 				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
-		if ret.ExitCode != 0 {
-			return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
-		}
 
 		return nil
 	}

chain/consensus/filcns/filecoin.go

@@ -91,9 +91,6 @@ var RewardFunc = func(ctx context.Context, vmi vm.Interface, em stmgr.ExecMonito
 		}
 	}
 
-	if ret.ExitCode != 0 {
-		return xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
-	}
 	return nil
 }
 

chain/events/events_height.go

@@ -180,7 +180,7 @@ func (e *heightEventsObserver) Revert(ctx context.Context, from, to *types.TipSe
 	// Update the head first so we don't accidental skip reverting a concurrent call to ChainAt.
 	e.updateHead(to)
 
-	// Call revert on all hights between the two tipsets, handling empty tipsets.
+	// Call revert on all heights between the two tipsets, handling empty tipsets.
 	for h := from.Height(); h > to.Height(); h-- {
 		e.lk.Lock()
 		triggers := e.tsHeights[h]

chain/events/events_test.go

@@ -673,7 +673,7 @@ func TestCalled(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing get's randomly called
+	// create few blocks to make sure nothing gets randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -991,7 +991,7 @@ func TestCalledNull(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing get's randomly called
+	// create few blocks to make sure nothing gets randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -1050,7 +1050,7 @@ func TestRemoveTriggersOnMessage(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing get's randomly called
+	// create few blocks to make sure nothing gets randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -1155,7 +1155,7 @@ func TestStateChanged(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing get's randomly called
+	// create few blocks to make sure nothing gets randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)

chain/events/filter/index.go

@@ -326,7 +326,7 @@ func NewEventIndex(ctx context.Context, path string, chainStore *store.ChainStor
 		}
 
 		if version == 2 {
-			log.Infof("upgrading event index from version 1 to version 2")
+			log.Infof("upgrading event index from version 2 to version 3")
 
 			// to upgrade to version 3 we only need to create an index on the event_entry.key column
 			// and on the event.emitter_addr column

chain/events/state/ctxstore.go

@@ -1,25 +0,0 @@
-package state
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	cbor "github.com/ipfs/go-ipld-cbor"
-)
-
-type contextStore struct {
-	ctx context.Context
-	cst *cbor.BasicIpldStore
-}
-
-func (cs *contextStore) Context() context.Context {
-	return cs.ctx
-}
-
-func (cs *contextStore) Get(ctx context.Context, c cid.Cid, out interface{}) error {
-	return cs.cst.Get(ctx, c, out)
-}
-
-func (cs *contextStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) {
-	return cs.cst.Put(ctx, v)
-}

chain/exchange/cbor_gen.go

@@ -306,9 +306,9 @@ func (t *Response) UnmarshalCBOR(r io.Reader) (err error) {
 	return nil
 }
 
-var lengthBufCompactedMessagesCBOR = []byte{132}
+var lengthBufCompactedMessages = []byte{132}
 
-func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
+func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
@@ -316,12 +316,12 @@ func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write(lengthBufCompactedMessagesCBOR); err != nil {
+	if _, err := cw.Write(lengthBufCompactedMessages); err != nil {
 		return err
 	}
 
 	// t.Bls ([]*types.Message) (slice)
-	if len(t.Bls) > 150000 {
+	if len(t.Bls) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Bls was too long")
 	}
 
@@ -334,7 +334,7 @@ func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.BlsIncludes ([]exchange.messageIndices) (slice)
+	// t.BlsIncludes ([][]uint64) (slice)
 	if len(t.BlsIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
 	}
@@ -343,13 +343,24 @@ func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.BlsIncludes {
-		if err := v.MarshalCBOR(cw); err != nil {
+		if len(v) > cbg.MaxLength {
+			return xerrors.Errorf("Slice value in field v was too long")
+		}
+
+		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
 			return err
 		}
+		for _, v := range v {
+
+			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
+				return err
+			}
+
+		}
 	}
 
 	// t.Secpk ([]*types.SignedMessage) (slice)
-	if len(t.Secpk) > 150000 {
+	if len(t.Secpk) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Secpk was too long")
 	}
 
@@ -362,7 +373,7 @@ func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
+	// t.SecpkIncludes ([][]uint64) (slice)
 	if len(t.SecpkIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
 	}
@@ -371,15 +382,26 @@ func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.SecpkIncludes {
-		if err := v.MarshalCBOR(cw); err != nil {
+		if len(v) > cbg.MaxLength {
+			return xerrors.Errorf("Slice value in field v was too long")
+		}
+
+		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
 			return err
 		}
+		for _, v := range v {
+
+			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
+				return err
+			}
+
+		}
 	}
 	return nil
 }
 
-func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = CompactedMessagesCBOR{}
+func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = CompactedMessages{}
 
 	cr := cbg.NewCborReader(r)
 
@@ -408,7 +430,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > 150000 {
+	if extra > cbg.MaxLength {
 		return fmt.Errorf("t.Bls: array too large (%d)", extra)
 	}
 
@@ -449,7 +471,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.BlsIncludes ([]exchange.messageIndices) (slice)
+	// t.BlsIncludes ([][]uint64) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -465,7 +487,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.BlsIncludes = make([]messageIndices, extra)
+		t.BlsIncludes = make([][]uint64, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -477,13 +499,47 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			{
-
-				if err := t.BlsIncludes[i].UnmarshalCBOR(cr); err != nil {
-					return xerrors.Errorf("unmarshaling t.BlsIncludes[i]: %w", err)
-				}
-
+			maj, extra, err = cr.ReadHeader()
+			if err != nil {
+				return err
 			}
+
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
+			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+
+			if extra > 0 {
+				t.BlsIncludes[i] = make([]uint64, extra)
+			}
+
+			for j := 0; j < int(extra); j++ {
+				{
+					var maj byte
+					var extra uint64
+					var err error
+					_ = maj
+					_ = extra
+					_ = err
+
+					{
+
+						maj, extra, err = cr.ReadHeader()
+						if err != nil {
+							return err
+						}
+						if maj != cbg.MajUnsignedInt {
+							return fmt.Errorf("wrong type for uint64 field")
+						}
+						t.BlsIncludes[i][j] = uint64(extra)
+
+					}
+				}
+			}
+
 		}
 	}
 
@@ -494,7 +550,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > 150000 {
+	if extra > cbg.MaxLength {
 		return fmt.Errorf("t.Secpk: array too large (%d)", extra)
 	}
 
@@ -535,7 +591,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
+	// t.SecpkIncludes ([][]uint64) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -551,7 +607,7 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.SecpkIncludes = make([]messageIndices, extra)
+		t.SecpkIncludes = make([][]uint64, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -563,13 +619,47 @@ func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			{
-
-				if err := t.SecpkIncludes[i].UnmarshalCBOR(cr); err != nil {
-					return xerrors.Errorf("unmarshaling t.SecpkIncludes[i]: %w", err)
-				}
-
+			maj, extra, err = cr.ReadHeader()
+			if err != nil {
+				return err
 			}
+
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
+			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+
+			if extra > 0 {
+				t.SecpkIncludes[i] = make([]uint64, extra)
+			}
+
+			for j := 0; j < int(extra); j++ {
+				{
+					var maj byte
+					var extra uint64
+					var err error
+					_ = maj
+					_ = extra
+					_ = err
+
+					{
+
+						maj, extra, err = cr.ReadHeader()
+						if err != nil {
+							return err
+						}
+						if maj != cbg.MajUnsignedInt {
+							return fmt.Errorf("wrong type for uint64 field")
+						}
+						t.SecpkIncludes[i][j] = uint64(extra)
+
+					}
+				}
+			}
+
 		}
 	}
 

chain/exchange/client.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 	"math/rand"
 	"time"
 
@@ -24,10 +23,6 @@ import (
 	"github.com/filecoin-project/lotus/lib/peermgr"
 )
 
-// Set the max exchange message size to 120MiB. Purely based on gas numbers, we can include ~8MiB of
-// messages per block, so I've set this to 120MiB to be _very_ safe.
-const maxExchangeMessageSize = (15 * 8) << 20
-
 // client implements exchange.Client, using the libp2p ChainExchange protocol
 // as the fetching mechanism.
 type client struct {
@@ -247,7 +242,7 @@ func (c *client) processResponse(req *Request, res *Response, tipsets []*types.T
 			// If we didn't request the headers they should have been provided
 			// by the caller.
 			if len(tipsets) < len(res.Chain) {
-				return nil, xerrors.Errorf("not enought tipsets provided for message response validation, needed %d, have %d", len(res.Chain), len(tipsets))
+				return nil, xerrors.Errorf("not enough tipsets provided for message response validation, needed %d, have %d", len(res.Chain), len(tipsets))
 			}
 			chain := make([]*BSTipSet, 0, resLength)
 			for i, resChain := range res.Chain {
@@ -386,7 +381,7 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 	defer span.End()
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
-			trace.StringAttribute("peer", peer.Pretty()),
+			trace.StringAttribute("peer", peer.String()),
 		)
 	}
 	defer func() {
@@ -439,11 +434,10 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 		log.Warnw("CloseWrite err", "error", err)
 	}
 
-	// Read response, limiting the size of the response to maxExchangeMessageSize as we allow a
-	// lot of messages (10k+) but they'll mostly be quite small.
+	// Read response.
 	var res Response
 	err = cborutil.ReadCborRPC(
-		bufio.NewReader(io.LimitReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline), maxExchangeMessageSize)),
+		bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)),
 		&res)
 	if err != nil {
 		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)

chain/exchange/protocol.go

@@ -154,8 +154,6 @@ type BSTipSet struct {
 // FIXME: The logic to decompress this structure should belong
 //
 //	to itself, not to the consumer.
-//
-// NOTE: Max messages is: BlockMessageLimit (10k) * MaxTipsetSize (15) = 150k
 type CompactedMessages struct {
 	Bls         []*types.Message
 	BlsIncludes [][]uint64

chain/exchange/protocol_encoding.go

@@ -1,125 +0,0 @@
-package exchange
-
-import (
-	"fmt"
-	"io"
-
-	cbg "github.com/whyrusleeping/cbor-gen"
-	xerrors "golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/build"
-	types "github.com/filecoin-project/lotus/chain/types"
-)
-
-// Type used for encoding/decoding compacted messages. This is a ustom type as we need custom limits.
-// - Max messages is 150,000 as that's 15 times the max block size (in messages). It needs to be
-// large enough to cover a full tipset full of full blocks.
-type CompactedMessagesCBOR struct {
-	Bls         []*types.Message `cborgen:"maxlen=150000"`
-	BlsIncludes []messageIndices
-
-	Secpk         []*types.SignedMessage `cborgen:"maxlen=150000"`
-	SecpkIncludes []messageIndices
-}
-
-// Unmarshal into the "decoding" struct, then copy into the actual struct.
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
-	var c CompactedMessagesCBOR
-	if err := c.UnmarshalCBOR(r); err != nil {
-		return err
-	}
-	t.Bls = c.Bls
-	t.BlsIncludes = make([][]uint64, len(c.BlsIncludes))
-	for i, v := range c.BlsIncludes {
-		t.BlsIncludes[i] = v.v
-	}
-	t.Secpk = c.Secpk
-	t.SecpkIncludes = make([][]uint64, len(c.SecpkIncludes))
-	for i, v := range c.SecpkIncludes {
-		t.SecpkIncludes[i] = v.v
-	}
-	return nil
-}
-
-// Copy into the encoding struct, then marshal.
-func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-
-	var c CompactedMessagesCBOR
-	c.Bls = t.Bls
-	c.BlsIncludes = make([]messageIndices, len(t.BlsIncludes))
-	for i, v := range t.BlsIncludes {
-		c.BlsIncludes[i].v = v
-	}
-	c.Secpk = t.Secpk
-	c.SecpkIncludes = make([]messageIndices, len(t.SecpkIncludes))
-	for i, v := range t.SecpkIncludes {
-		c.SecpkIncludes[i].v = v
-	}
-	return c.MarshalCBOR(w)
-}
-
-// this needs to be a struct or cborgen will peak into it and ignore the Unmarshal/Marshal functions
-type messageIndices struct {
-	v []uint64
-}
-
-func (t *messageIndices) UnmarshalCBOR(r io.Reader) (err error) {
-	cr := cbg.NewCborReader(r)
-
-	maj, extra, err := cr.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra > uint64(build.BlockMessageLimit) {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	if extra > 0 {
-		t.v = make([]uint64, extra)
-	}
-
-	for i := 0; i < int(extra); i++ {
-		maj, extra, err := cr.ReadHeader()
-		if err != nil {
-			return err
-		}
-		if maj != cbg.MajUnsignedInt {
-			return fmt.Errorf("wrong type for uint64 field")
-		}
-		t.v[i] = extra
-
-	}
-	return nil
-}
-
-func (t *messageIndices) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-
-	cw := cbg.NewCborWriter(w)
-
-	if len(t.v) > build.BlockMessageLimit {
-		return xerrors.Errorf("Slice value in field v was too long")
-	}
-
-	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.v))); err != nil {
-		return err
-	}
-	for _, v := range t.v {
-		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, v); err != nil {
-			return err
-		}
-	}
-	return nil
-}

chain/index/msgindex.go

@@ -131,7 +131,7 @@ func NewMsgIndex(lctx context.Context, basePath string, cs ChainStore) (MsgIndex
 
 	db, err := sql.Open("sqlite3", dbPath)
 	if err != nil {
-		// TODO [nice to have]: automaticaly delete corrupt databases
+		// TODO [nice to have]: automatically delete corrupt databases
 		//      but for now we can just error and let the operator delete.
 		return nil, xerrors.Errorf("error opening msgindex database: %w", err)
 	}

chain/market/store.go

@@ -39,23 +39,6 @@ func (ps *Store) save(ctx context.Context, state *FundedAddressState) error {
 	return ps.ds.Put(ctx, k, b)
 }
 
-// get the state for the given address
-func (ps *Store) get(ctx context.Context, addr address.Address) (*FundedAddressState, error) {
-	k := dskeyForAddr(addr)
-
-	data, err := ps.ds.Get(ctx, k)
-	if err != nil {
-		return nil, err
-	}
-
-	var state FundedAddressState
-	err = cborrpc.ReadCborRPC(bytes.NewReader(data), &state)
-	if err != nil {
-		return nil, err
-	}
-	return &state, nil
-}
-
 // forEach calls iter with each address in the datastore
 func (ps *Store) forEach(ctx context.Context, iter func(*FundedAddressState)) error {
 	res, err := ps.ds.Query(ctx, dsq.Query{Prefix: dsKeyAddr})

chain/messagepool/messagepool.go

@@ -21,6 +21,7 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/minio/blake2b-simd"
 	"github.com/raulk/clock"
+	"go.opencensus.io/stats"
 	"golang.org/x/xerrors"
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
@@ -210,8 +211,10 @@ func ComputeRBF(curPrem abi.TokenAmount, replaceByFeeRatio types.Percent) abi.To
 
 func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.MessageSendSpec) {
 	var maxFee abi.TokenAmount
+	var maximizeFeeCap bool
 	if sendSpec != nil {
 		maxFee = sendSpec.MaxFee
+		maximizeFeeCap = sendSpec.MaximizeFeeCap
 	}
 	if maxFee.Int == nil || maxFee.Equals(big.Zero()) {
 		mf, err := mff()
@@ -222,15 +225,12 @@ func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.M
 		maxFee = mf
 	}
 
-	gl := types.NewInt(uint64(msg.GasLimit))
-	totalFee := types.BigMul(msg.GasFeeCap, gl)
-
-	if totalFee.LessThanEqual(maxFee) {
-		msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
-		return
+	gaslimit := types.NewInt(uint64(msg.GasLimit))
+	totalFee := types.BigMul(msg.GasFeeCap, gaslimit)
+	if maximizeFeeCap || totalFee.GreaterThan(maxFee) {
+		msg.GasFeeCap = big.Div(maxFee, gaslimit)
 	}
 
-	msg.GasFeeCap = big.Div(maxFee, gl)
 	msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
 }
 
@@ -1022,6 +1022,9 @@ func (mp *MessagePool) addLocked(ctx context.Context, m *types.SignedMessage, st
 		}
 	})
 
+	// Record the current size of the Mpool
+	stats.Record(ctx, metrics.MpoolMessageCount.M(int64(mp.currentSize)))
+
 	return nil
 }
 
@@ -1214,6 +1217,9 @@ func (mp *MessagePool) remove(ctx context.Context, from address.Address, nonce u
 			return
 		}
 	}
+
+	// Record the current size of the Mpool
+	stats.Record(ctx, metrics.MpoolMessageCount.M(int64(mp.currentSize)))
 }
 
 func (mp *MessagePool) Pending(ctx context.Context) ([]*types.SignedMessage, *types.TipSet) {

chain/state/statetree.go

@@ -570,7 +570,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error
 			}
 
 			// no need to record anything here, there are no duplicates in the actors HAMT
-			// iself.
+			// itself.
 			if _, ok := seen[addr]; ok {
 				return nil
 			}
@@ -588,7 +588,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error
 		}
 
 		// no need to record anything here, there are no duplicates in the actors HAMT
-		// iself.
+		// itself.
 		if _, ok := seen[addr]; ok {
 			return nil
 		}

chain/stmgr/forks.go

@@ -235,11 +235,6 @@ func (sm *StateManager) hasExpensiveForkBetween(parent, height abi.ChainEpoch) b
 	return false
 }
 
-func (sm *StateManager) hasExpensiveFork(height abi.ChainEpoch) bool {
-	_, ok := sm.expensiveUpgrades[height]
-	return ok
-}
-
 func runPreMigration(ctx context.Context, sm *StateManager, fn PreMigrationFunc, cache *nv16.MemMigrationCache, ts *types.TipSet) {
 	height := ts.Height()
 	parent := ts.ParentState()

chain/store/index_test.go

@@ -42,7 +42,7 @@ func TestIndexSeeks(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	_, err = cs.Import(ctx, bytes.NewReader(gencar))
+	_, _, err = cs.Import(ctx, bytes.NewReader(gencar))
 	if err != nil {
 		t.Fatal(err)
 	}

chain/store/messages.go

@@ -212,13 +212,8 @@ func (cs *ChainStore) MessagesForTipset(ctx context.Context, ts *types.TipSet) (
 
 	var out []types.ChainMsg
 	for _, bm := range bmsgs {
-		for _, blsm := range bm.BlsMessages {
-			out = append(out, blsm)
-		}
-
-		for _, secm := range bm.SecpkMessages {
-			out = append(out, secm)
-		}
+		out = append(out, bm.BlsMessages...)
+		out = append(out, bm.SecpkMessages...)
 	}
 
 	return out, nil

chain/store/snapshot.go

@@ -60,7 +60,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 	})
 }
 
-func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, error) {
+func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipSet, genesis *types.BlockHeader, err error) {
 	// TODO: writing only to the state blockstore is incorrect.
 	//  At this time, both the state and chain blockstores are backed by the
 	//  universal store. When we physically segregate the stores, we will need
@@ -69,7 +69,7 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, e
 
 	br, err := carv2.NewBlockReader(r)
 	if err != nil {
-		return nil, xerrors.Errorf("loadcar failed: %w", err)
+		return nil, nil, xerrors.Errorf("loadcar failed: %w", err)
 	}
 
 	s := cs.StateBlockstore()
@@ -80,27 +80,51 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, e
 		putThrottle <- nil
 	}
 
+	if len(br.Roots) == 0 {
+		return nil, nil, xerrors.Errorf("no roots in snapshot car file")
+	}
+	nextTailCid := br.Roots[0]
+
+	var tailBlock types.BlockHeader
+	tailBlock.Height = abi.ChainEpoch(-1)
+
 	var buf []blocks.Block
 	for {
 		blk, err := br.Next()
 		if err != nil {
+
+			// we're at the end
 			if err == io.EOF {
 				if len(buf) > 0 {
 					if err := s.PutMany(ctx, buf); err != nil {
-						return nil, err
+						return nil, nil, err
 					}
 				}
 
 				break
 			}
-			return nil, err
+			return nil, nil, err
 		}
 
+		// check for header block, looking for genesis
+		if blk.Cid() == nextTailCid && tailBlock.Height != 0 {
+			if err := tailBlock.UnmarshalCBOR(bytes.NewReader(blk.RawData())); err != nil {
+				return nil, nil, xerrors.Errorf("failed to unmarshal genesis block: %w", err)
+			}
+			if len(tailBlock.Parents) > 0 {
+				nextTailCid = tailBlock.Parents[0]
+			} else {
+				// note: even the 0th block has a parent linking to the cbor genesis block
+				return nil, nil, xerrors.Errorf("current block (epoch %d cid %s) has no parents", tailBlock.Height, tailBlock.Cid())
+			}
+		}
+
+		// append to batch
 		buf = append(buf, blk)
 
 		if len(buf) > 1000 {
 			if lastErr := <-putThrottle; lastErr != nil { // consume one error to have the right to add one
-				return nil, lastErr
+				return nil, nil, lastErr
 			}
 
 			go func(buf []blocks.Block) {
@@ -113,13 +137,17 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, e
 	// check errors
 	for i := 0; i < parallelPuts; i++ {
 		if lastErr := <-putThrottle; lastErr != nil {
-			return nil, lastErr
+			return nil, nil, lastErr
 		}
 	}
 
+	if tailBlock.Height != 0 {
+		return nil, nil, xerrors.Errorf("expected genesis block to have height 0 (genesis), got %d: %s", tailBlock.Height, tailBlock.Cid())
+	}
+
 	root, err := cs.LoadTipSet(ctx, types.NewTipSetKey(br.Roots...))
 	if err != nil {
-		return nil, xerrors.Errorf("failed to load root tipset from chainfile: %w", err)
+		return nil, nil, xerrors.Errorf("failed to load root tipset from chainfile: %w", err)
 	}
 
 	ts := root
@@ -135,10 +163,10 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, e
 	}
 
 	if err := cs.PersistTipsets(ctx, tssToPersist); err != nil {
-		return nil, xerrors.Errorf("failed to persist tipsets: %w", err)
+		return nil, nil, xerrors.Errorf("failed to persist tipsets: %w", err)
 	}
 
-	return root, nil
+	return root, &tailBlock, nil
 }
 
 type walkSchedTaskType int
@@ -167,7 +195,7 @@ func (t walkSchedTaskType) String() string {
 	case dagTask:
 		return "dag"
 	}
-	panic(fmt.Sprintf("unknow task %d", t))
+	panic(fmt.Sprintf("unknown task %d", t))
 }
 
 type walkTask struct {
@@ -656,9 +684,7 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 		}
 
 		if b.Height > 0 {
-			for _, p := range b.Parents {
-				blocksToWalk = append(blocksToWalk, p)
-			}
+			blocksToWalk = append(blocksToWalk, b.Parents...)
 		} else {
 			// include the genesis block
 			cids = append(cids, b.Parents...)

chain/store/store_test.go

@@ -118,7 +118,7 @@ func TestChainExportImport(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, err := cs.Import(context.TODO(), buf)
+	root, _, err := cs.Import(context.TODO(), buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +153,7 @@ func TestChainImportTipsetKeyCid(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, err := cs.Import(ctx, buf)
+	root, _, err := cs.Import(ctx, buf)
 	require.NoError(t, err)
 
 	require.Truef(t, root.Equals(last), "imported chain differed from exported chain")
@@ -202,7 +202,7 @@ func TestChainExportImportFull(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, ds, filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, err := cs.Import(context.TODO(), buf)
+	root, _, err := cs.Import(context.TODO(), buf)
 	if err != nil {
 		t.Fatal(err)
 	}

chain/sub/incoming.go

@@ -516,7 +516,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 		return pubsub.ValidationReject
 	}
 	if len(idxrMsg.ExtraData) == 0 {
-		log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
+		log.Debugw("ignoring message missing miner id", "peer", originPeer)
 		return pubsub.ValidationIgnore
 	}
 
@@ -552,7 +552,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 		// Check that the miner ID maps to the peer that sent the message.
 		err = v.authenticateMessage(ctx, minerAddr, originPeer)
 		if err != nil {
-			log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerAddr)
+			log.Warnw("cannot authenticate message", "err", err, "peer", originPeer, "minerID", minerAddr)
 			stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
 			return pubsub.ValidationReject
 		}

chain/sub/ratelimit/queue_test.go

@@ -0,0 +1,61 @@
+package ratelimit
+
+import (
+	"testing"
+)
+
+func TestQueue(t *testing.T) {
+	const size = 3
+	q := &queue{buf: make([]int64, size)}
+
+	if q.len() != 0 {
+		t.Fatalf("q.len() = %d, expect 0", q.len())
+	}
+
+	if q.cap() != size {
+		t.Fatalf("q.cap() = %d, expect %d", q.cap(), size)
+	}
+
+	for i := int64(0); i < int64(size); i++ {
+		err := q.push(i)
+		if err != nil {
+			t.Fatalf("cannot push element %d", i)
+		}
+	}
+
+	if q.len() != size {
+		t.Fatalf("q.len() = %d, expect %d", q.len(), size)
+	}
+
+	err := q.push(int64(size))
+	if err != ErrRateLimitExceeded {
+		t.Fatalf("pushing element beyond capacity should have failed with err: %s, got %s", ErrRateLimitExceeded, err)
+	}
+
+	if q.front() != 0 {
+		t.Fatalf("q.front() = %d, expect 0", q.front())
+	}
+
+	if q.back() != int64(size-1) {
+		t.Fatalf("q.back() = %d, expect %d", q.back(), size-1)
+	}
+
+	popVal := q.pop()
+	if popVal != 0 {
+		t.Fatalf("q.pop() = %d, expect 0", popVal)
+	}
+
+	if q.len() != size-1 {
+		t.Fatalf("q.len() = %d, expect %d", q.len(), size-1)
+	}
+
+	// Testing truncation.
+	threshold := int64(1)
+	q.truncate(threshold)
+	if q.len() != 1 {
+		t.Fatalf("q.len() after truncate = %d, expect 1", q.len())
+	}
+	if q.front() != 2 {
+		t.Fatalf("q.front() after truncate = %d, expect 2", q.front())
+	}
+}

chain/sync.go

@@ -844,7 +844,7 @@ loop:
 		return nil, xerrors.Errorf("failed to load next local tipset: %w", err)
 	}
 	if base.IsChildOf(knownParent) {
-		// common case: receiving a block thats potentially part of the same tipset as our best block
+		// common case: receiving a block that's potentially part of the same tipset as our best block
 		return blockSet, nil
 	}
 

chain/sync_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/libp2p/go-libp2p/core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
 
@@ -344,13 +343,6 @@ func (tu *syncTestUtil) addClientNode() int {
 	return len(tu.nds) - 1
 }
 
-func (tu *syncTestUtil) pid(n int) peer.ID {
-	nal, err := tu.nds[n].NetAddrsListen(tu.ctx)
-	require.NoError(tu.t, err)
-
-	return nal.ID
-}
-
 func (tu *syncTestUtil) connect(from, to int) {
 	toPI, err := tu.nds[to].NetAddrsListen(tu.ctx)
 	require.NoError(tu.t, err)

chain/types/blockmsg_test.go

@@ -7,9 +7,6 @@ import (
 )
 
 func TestDecodeBlockMsg(t *testing.T) {
-	type args struct {
-		b []byte
-	}
 	tests := []struct {
 		name    string
 		data    []byte

chain/types/ethtypes/eth_transactions.go

@@ -62,9 +62,14 @@ type EthTxArgs struct {
 // - BlockHash
 // - BlockNumber
 // - TransactionIndex
-// - From
 // - Hash
 func EthTxFromSignedEthMessage(smsg *types.SignedMessage) (EthTx, error) {
+	// The from address is always an f410f address, never an ID or other address.
+	if !IsEthAddress(smsg.Message.From) {
+		return EthTx{}, xerrors.Errorf("sender must be an eth account, was %s", smsg.Message.From)
+	}
+
+	// Probably redundant, but we might as well check.
 	if smsg.Signature.Type != typescrypto.SigTypeDelegated {
 		return EthTx{}, xerrors.Errorf("signature is not delegated type, is type: %d", smsg.Signature.Type)
 	}
@@ -79,10 +84,18 @@ func EthTxFromSignedEthMessage(smsg *types.SignedMessage) (EthTx, error) {
 		return EthTx{}, xerrors.Errorf("failed to recover signature: %w", err)
 	}
 
+	from, err := EthAddressFromFilecoinAddress(smsg.Message.From)
+	if err != nil {
+		// This should be impossible as we've already asserted that we have an EthAddress
+		// sender...
+		return EthTx{}, xerrors.Errorf("sender was not an eth account")
+	}
+
 	return EthTx{
 		Nonce:                EthUint64(txArgs.Nonce),
 		ChainID:              EthUint64(txArgs.ChainID),
 		To:                   txArgs.To,
+		From:                 from,
 		Value:                EthBigInt(txArgs.Value),
 		Type:                 Eip1559TxType,
 		Gas:                  EthUint64(txArgs.GasLimit),

chain/types/ethtypes/eth_types.go

@@ -799,6 +799,45 @@ func GetContractEthAddressFromCode(sender EthAddress, salt [32]byte, initcode []
 	return ethAddr, nil
 }
 
+// EthEstimateGasParams handles raw jsonrpc params for eth_estimateGas
+type EthEstimateGasParams struct {
+	Tx       EthCall
+	BlkParam *EthBlockNumberOrHash
+}
+
+func (e *EthEstimateGasParams) UnmarshalJSON(b []byte) error {
+	var params []json.RawMessage
+	err := json.Unmarshal(b, &params)
+	if err != nil {
+		return err
+	}
+
+	switch len(params) {
+	case 2:
+		err = json.Unmarshal(params[1], &e.BlkParam)
+		if err != nil {
+			return err
+		}
+		fallthrough
+	case 1:
+		err = json.Unmarshal(params[0], &e.Tx)
+		if err != nil {
+			return err
+		}
+	default:
+		return xerrors.Errorf("expected 1 or 2 params, got %d", len(params))
+	}
+
+	return nil
+}
+
+func (e EthEstimateGasParams) MarshalJSON() ([]byte, error) {
+	if e.BlkParam != nil {
+		return json.Marshal([]interface{}{e.Tx, e.BlkParam})
+	}
+	return json.Marshal([]interface{}{e.Tx})
+}
+
 // EthFeeHistoryParams handles raw jsonrpc params for eth_feeHistory
 type EthFeeHistoryParams struct {
 	BlkCount          EthUint64

chain/types/message_receipt_cbor.go

@@ -140,7 +140,7 @@ func (t *messageReceiptV0) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative oveflow")
+				return fmt.Errorf("int64 negative overflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -186,7 +186,7 @@ func (t *messageReceiptV0) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative oveflow")
+				return fmt.Errorf("int64 negative overflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -278,7 +278,7 @@ func (t *messageReceiptV1) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative oveflow")
+				return fmt.Errorf("int64 negative overflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -324,7 +324,7 @@ func (t *messageReceiptV1) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative oveflow")
+				return fmt.Errorf("int64 negative overflow")
 			}
 			extraI = -1 - extraI
 		default:

chain/types/vmcontext.go

@@ -27,24 +27,3 @@ type StateTree interface {
 
 	Version() StateTreeVersion
 }
-
-type storageWrapper struct {
-	s Storage
-}
-
-func (sw *storageWrapper) Put(i cbg.CBORMarshaler) (cid.Cid, error) {
-	c, err := sw.s.Put(i)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	return c, nil
-}
-
-func (sw *storageWrapper) Get(c cid.Cid, out cbg.CBORUnmarshaler) error {
-	if err := sw.s.Get(c, out); err != nil {
-		return err
-	}
-
-	return nil
-}

cli/net.go

@@ -282,7 +282,7 @@ var NetDisconnect = &cli.Command{
 				fmt.Println("failure")
 				return err
 			}
-			fmt.Printf("disconnect %s: ", pid.Pretty())
+			fmt.Printf("disconnect %s: ", pid)
 			err = api.NetDisconnect(ctx, pid)
 			if err != nil {
 				fmt.Println("failure")
@@ -312,7 +312,7 @@ var NetConnect = &cli.Command{
 		}
 
 		for _, pi := range pis {
-			fmt.Printf("connect %s: ", pi.ID.Pretty())
+			fmt.Printf("connect %s: ", pi.ID)
 			err := api.NetConnect(ctx, pi)
 			if err != nil {
 				fmt.Println("failure")

cli/state.go

@@ -1920,8 +1920,29 @@ var StateSysActorCIDsCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		for name, cid := range actorsCids {
-			_, _ = fmt.Fprintf(tw, "%v\t%v\n", name, cid)
+
+		var actorsCidTuples []struct {
+			actorName string
+			actorCid  cid.Cid
+		}
+
+		for name, actorCid := range actorsCids {
+			keyVal := struct {
+				actorName string
+				actorCid  cid.Cid
+			}{
+				actorName: name,
+				actorCid:  actorCid,
+			}
+			actorsCidTuples = append(actorsCidTuples, keyVal)
+		}
+
+		sort.Slice(actorsCidTuples, func(i, j int) bool {
+			return actorsCidTuples[i].actorName < actorsCidTuples[j].actorName
+		})
+
+		for _, keyVal := range actorsCidTuples {
+			_, _ = fmt.Fprintf(tw, "%v\t%v\n", keyVal.actorName, keyVal.actorCid)
 		}
 		return tw.Flush()
 	},

cli/sync.go

@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 )
 
 var SyncCmd = &cli.Command{
@@ -262,6 +263,9 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 	}
 	firstApp = state.VMApplied
 
+	// eta computes the ETA for the sync to complete (with a lookback of 10 processed items)
+	eta := cliutil.NewETA(10)
+
 	for {
 		state, err := napi.SyncState(ctx)
 		if err != nil {
@@ -312,8 +316,10 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 			fmt.Print("\r\x1b[2K\x1b[A")
 		}
 
+		todo := theight - ss.Height
+
 		fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", workerID, baseHeight, theight, heightDiff)
-		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d\n", ss.Stage, ss.Height, theight-ss.Height)
+		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d, ETA: %s\n", ss.Stage, ss.Height, todo, eta.Update(int64(todo)))
 		lastLines = 2
 
 		if i%samples == 0 {

cli/util/eta.go

@@ -0,0 +1,94 @@
+package cliutil
+
+import (
+	"fmt"
+	"math"
+	"time"
+)
+
+// ETA implements a very simple eta calculator based on the number of remaining items. It does not
+// require knowing the work size in advance and is therefore suitable for streaming workloads and
+// also does not require that consecutive updates have a monotonically decreasing remaining value.
+type ETA struct {
+	// max number of items to keep in memory
+	maxItems int
+	// a queue of most recently updated items
+	items []item
+	// we store the last calculated ETA which we reuse if there was not change in remaining items
+	lastETA string
+}
+
+type item struct {
+	timestamp time.Time
+	remaining int64
+}
+
+// NewETA creates a new ETA calculator of the given size
+func NewETA(maxItems int) *ETA {
+	return &ETA{
+		maxItems: maxItems,
+		items:    make([]item, 0),
+	}
+}
+
+// Update updates the ETA calculator with the remaining number of items and returns the ETA
+func (e *ETA) Update(remaining int64) string {
+	item := item{
+		timestamp: time.Now(),
+		remaining: remaining,
+	}
+
+	if len(e.items) == 0 {
+		e.items = append(e.items, item)
+		return ""
+	}
+
+	if e.items[len(e.items)-1].remaining == remaining {
+		// we ignore updates with the same remaining value and just return the previous ETA
+		return e.lastETA
+	} else if e.items[len(e.items)-1].remaining < remaining {
+		// remaining went up from previous update, lets estimate how many items were processed using the
+		// average number processed items in the queue.
+		var avgProcessedPerItem int64 = 1
+		if len(e.items) > 1 {
+			diffRemaining := e.items[0].remaining - e.items[len(e.items)-1].remaining
+			avgProcessedPerItem = int64(math.Round(float64(diffRemaining) / float64(len(e.items))))
+		}
+
+		// diff is the difference in increase in remaining since last update plus the average number of processed
+		// items we estimate that were processed this round
+		diff := remaining - e.items[len(e.items)-1].remaining + avgProcessedPerItem
+
+		// we update all items in the queue by shifting their remaining value accordingly. This means that we
+		// always have strictly decreasing remaining values in the queue
+		for i := range e.items {
+			e.items[i].remaining += diff
+		}
+	}
+
+	// append the item to the queue and remove the oldest item if needed
+	if len(e.items) >= e.maxItems {
+		e.items = e.items[1:]
+	}
+	e.items = append(e.items, item)
+
+	// calculate the average processing time per item in the queue
+	diffMs := e.items[len(e.items)-1].timestamp.Sub(e.items[0].timestamp).Milliseconds()
+	nrItemsProcessed := e.items[0].remaining - e.items[len(e.items)-1].remaining
+	avg := diffMs / nrItemsProcessed
+
+	// use that average processing time to estimate how long the remaining items will take
+	// and cache that ETA so we don't have to recalculate it on every update unless the
+	// remaining value changes
+	e.lastETA = msToETA(avg * remaining)
+
+	return e.lastETA
+}
+
+func msToETA(ms int64) string {
+	seconds := ms / 1000
+	minutes := seconds / 60
+	hours := minutes / 60
+
+	return fmt.Sprintf("%02dh:%02dm:%02ds", hours, minutes%60, seconds%60)
+}

cli/wallet.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/tablewriter"
 )
@@ -459,7 +460,12 @@ var walletSign = &cli.Command{
 		sig, err := api.WalletSign(ctx, addr, msg)
 
 		if err != nil {
-			return err
+			// Check if the address is a multisig address
+			act, actErr := api.StateGetActor(ctx, addr, types.EmptyTSK)
+			if actErr == nil && builtin.IsMultisigActor(act.Code) {
+				return xerrors.Errorf("specified signer address is a multisig actor, it doesn’t have keys to sign transactions. To send a message with a multisig, signers of the multisig need to propose and approve transactions.")
+			}
+			return xerrors.Errorf("failed to sign message: %w", err)
 		}
 
 		sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)

cmd/lotus-bench/cli.go

@@ -0,0 +1,312 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cliCmd = &cli.Command{
+	Name:  "cli",
+	Usage: "Runs a concurrent stress test on one or more binaries commands and prints the performance metrics including latency distribution and histogram",
+	Description: `This benchmark has the following features:
+* Can query each command both sequentially and concurrently
+* Supports rate limiting
+* Can query multiple different commands at once (supporting different concurrency level and rate limiting for each command)
+* Gives a nice reporting summary of the stress testing of each command (including latency distribution, histogram and more)
+* Easy to use
+
+To use this benchmark you must specify the commands you want to test using the --cmd options, the format of it is:
+
+  --cmd=CMD[:CONCURRENCY][:QPS] where only NAME is required.
+
+Here are some real examples:
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' // runs the command with default concurrency and qps
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3'  // override concurrency to 3
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages::100' // override to 100 qps while using default concurrency
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3:100' // run using 3 workers but limit to 100 qps
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' --cmd='lotus sync wait' // run two commands at once
+`,
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name:  "duration",
+			Value: 60 * time.Second,
+			Usage: "Duration of benchmark in seconds",
+		},
+		&cli.IntFlag{
+			Name:  "concurrency",
+			Value: 10,
+			Usage: "How many workers should be used per command (can be overridden per command)",
+		},
+		&cli.IntFlag{
+			Name:  "qps",
+			Value: 0,
+			Usage: "How many requests per second should be sent per command (can be overridden per command), a value of 0 means no limit",
+		},
+		&cli.StringSliceFlag{
+			Name:  "cmd",
+			Usage: `Command to benchmark, you can specify multiple commands by repeating this flag. You can also specify command specific options to set the concurrency and qps for each command (see usage).`,
+		},
+		&cli.DurationFlag{
+			Name:  "watch",
+			Value: 0 * time.Second,
+			Usage: "If >0 then generates reports every N seconds (only supports linux/unix)",
+		},
+		&cli.BoolFlag{
+			Name:  "print-response",
+			Value: false,
+			Usage: "print the response of each request",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if len(cctx.StringSlice("cmd")) == 0 {
+			return errors.New("you must specify and least one cmd to benchmark")
+		}
+
+		var cmds []*CMD
+		for _, str := range cctx.StringSlice("cmd") {
+			entries := strings.SplitN(str, ":", 3)
+			if len(entries) == 0 {
+				return errors.New("invalid cmd format")
+			}
+
+			// check if concurrency was specified
+			concurrency := cctx.Int("concurrency")
+			if len(entries) > 1 {
+				if len(entries[1]) > 0 {
+					var err error
+					concurrency, err = strconv.Atoi(entries[1])
+					if err != nil {
+						return fmt.Errorf("could not parse concurrency value from command %s: %v", entries[0], err)
+					}
+				}
+			}
+
+			// check if qps was specified
+			qps := cctx.Int("qps")
+			if len(entries) > 2 {
+				if len(entries[2]) > 0 {
+					var err error
+					qps, err = strconv.Atoi(entries[2])
+					if err != nil {
+						return fmt.Errorf("could not parse qps value from command %s: %v", entries[0], err)
+					}
+				}
+			}
+
+			cmds = append(cmds, &CMD{
+				w:           os.Stdout,
+				cmd:         entries[0],
+				concurrency: concurrency,
+				qps:         qps,
+				printResp:   cctx.Bool("print-response"),
+			})
+		}
+
+		// terminate early on ctrl+c
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+		go func() {
+			<-c
+			fmt.Println("Received interrupt, stopping...")
+			for _, cmd := range cmds {
+				cmd.Stop()
+			}
+		}()
+
+		// stop all threads after duration
+		go func() {
+			time.Sleep(cctx.Duration("duration"))
+			for _, cmd := range cmds {
+				cmd.Stop()
+			}
+		}()
+
+		// start all threads
+		var wg sync.WaitGroup
+		wg.Add(len(cmds))
+
+		for _, cmd := range cmds {
+			go func(cmd *CMD) {
+				defer wg.Done()
+				err := cmd.Run()
+				if err != nil {
+					fmt.Printf("error running cmd: %v\n", err)
+				}
+			}(cmd)
+		}
+
+		// if watch is set then print a report every N seconds
+		var progressCh chan struct{}
+		if cctx.Duration("watch") > 0 {
+			progressCh = make(chan struct{}, 1)
+			go func(progressCh chan struct{}) {
+				ticker := time.NewTicker(cctx.Duration("watch"))
+				for {
+					clearAndPrintReport := func() {
+						// clear the screen move the cursor to the top left
+						fmt.Print("\033[2J")
+						fmt.Printf("\033[%d;%dH", 1, 1)
+						for i, cmd := range cmds {
+							cmd.Report()
+							if i < len(cmds)-1 {
+								fmt.Println()
+							}
+						}
+					}
+					select {
+					case <-ticker.C:
+						clearAndPrintReport()
+					case <-progressCh:
+						clearAndPrintReport()
+						return
+					}
+				}
+			}(progressCh)
+		}
+
+		wg.Wait()
+
+		if progressCh != nil {
+			// wait for the watch go routine to return
+			progressCh <- struct{}{}
+
+			// no need to print the report again
+			return nil
+		}
+
+		// print the report for each command
+		for i, cmd := range cmds {
+			cmd.Report()
+			if i < len(cmds)-1 {
+				fmt.Println()
+			}
+		}
+
+		return nil
+	},
+}
+
+// CMD handles the benchmarking of a single command.
+type CMD struct {
+	w io.Writer
+	// the cmd we want to benchmark
+	cmd string
+	// the number of concurrent requests to make to this command
+	concurrency int
+	// if >0 then limit to qps is the max number of requests per second to make to this command (0 = no limit)
+	qps int
+	// whether or not to print the response of each request (useful for debugging)
+	printResp bool
+	// instruct the worker go routines to stop
+	stopCh chan struct{}
+	// when the command bencharking started
+	start time.Time
+	// results channel is used by the workers to send results to the reporter
+	results chan *result
+	// reporter handles reading the results from workers and printing the report statistics
+	reporter *Reporter
+}
+
+func (c *CMD) Run() error {
+	var wg sync.WaitGroup
+	wg.Add(c.concurrency)
+
+	c.results = make(chan *result, c.concurrency*1_000)
+	c.stopCh = make(chan struct{}, c.concurrency)
+
+	go func() {
+		c.reporter = NewReporter(c.results, c.w)
+		c.reporter.Run()
+	}()
+
+	c.start = time.Now()
+
+	// throttle the number of requests per second
+	var qpsTicker *time.Ticker
+	if c.qps > 0 {
+		qpsTicker = time.NewTicker(time.Second / time.Duration(c.qps))
+	}
+
+	for i := 0; i < c.concurrency; i++ {
+		go func() {
+			c.startWorker(qpsTicker)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	// close the results channel so reporter will stop
+	close(c.results)
+
+	// wait until the reporter is done
+	<-c.reporter.doneCh
+
+	return nil
+}
+
+func (c *CMD) startWorker(qpsTicker *time.Ticker) {
+	for {
+		// check if we should stop
+		select {
+		case <-c.stopCh:
+			return
+		default:
+		}
+
+		// wait for the next tick if we are rate limiting this command
+		if qpsTicker != nil {
+			<-qpsTicker.C
+		}
+
+		start := time.Now()
+
+		var statusCode int = 0
+
+		arr := strings.Fields(c.cmd)
+
+		data, err := exec.Command(arr[0], arr[1:]...).Output()
+		if err != nil {
+			fmt.Println("1")
+			if exitError, ok := err.(*exec.ExitError); ok {
+				statusCode = exitError.ExitCode()
+			} else {
+				statusCode = 1
+			}
+		} else {
+			if c.printResp {
+				fmt.Printf("[%s] %s", c.cmd, string(data))
+			}
+		}
+
+		c.results <- &result{
+			statusCode: &statusCode,
+			err:        err,
+			duration:   time.Since(start),
+		}
+	}
+}
+
+func (c *CMD) Stop() {
+	for i := 0; i < c.concurrency; i++ {
+		c.stopCh <- struct{}{}
+	}
+}
+
+func (c *CMD) Report() {
+	total := time.Since(c.start)
+	fmt.Fprintf(c.w, "[%s]:\n", c.cmd)
+	fmt.Fprintf(c.w, "- Options:\n")
+	fmt.Fprintf(c.w, "  - concurrency: %d\n", c.concurrency)
+	fmt.Fprintf(c.w, "  - qps: %d\n", c.qps)
+	c.reporter.Print(total, c.w)
+}

cmd/lotus-bench/import.go

@@ -304,7 +304,7 @@ var importBenchCmd = &cli.Command{
 				return fmt.Errorf("no CAR file provided for import")
 			}
 
-			head, err = cs.Import(cctx.Context, carFile)
+			head, _, err = cs.Import(cctx.Context, carFile)
 			if err != nil {
 				return err
 			}
@@ -497,21 +497,6 @@ type Invocation struct {
 
 const GasPerNs = 10
 
-func countGasCosts(et *types.ExecutionTrace) int64 {
-	var cgas int64
-
-	for _, gc := range et.GasCharges {
-		cgas += gc.ComputeGas
-	}
-
-	for _, sub := range et.Subcalls {
-		c := countGasCosts(&sub) //nolint
-		cgas += c
-	}
-
-	return cgas
-}
-
 type stats struct {
 	timeTaken meanVar
 	gasRatio  meanVar

cmd/lotus-bench/main.go

@@ -120,6 +120,7 @@ func main() {
 			sealBenchCmd,
 			simpleCmd,
 			importBenchCmd,
+			cliCmd,
 			rpcCmd,
 		},
 	}

cmd/lotus-bench/reporter.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+)
+
+// result is the result of a single rpc method request.
+type result struct {
+	err        error
+	statusCode *int
+	duration   time.Duration
+}
+
+// Reporter reads the results from the workers through the results channel and aggregates the results.
+type Reporter struct {
+	// write the report to this writer
+	w io.Writer
+	// the reporter read the results from this channel
+	results chan *result
+	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
+	doneCh chan bool
+
+	// lock protect the following fields during critical sections (if --watch was specified)
+	lock sync.Mutex
+	// the latencies of all requests
+	latencies []int64
+	// the number of requests that returned each status code
+	statusCodes map[int]int
+	// the number of errors that occurred
+	errors map[string]int
+}
+
+func NewReporter(results chan *result, w io.Writer) *Reporter {
+	return &Reporter{
+		w:           w,
+		results:     results,
+		doneCh:      make(chan bool, 1),
+		statusCodes: make(map[int]int),
+		errors:      make(map[string]int),
+	}
+}
+
+func (r *Reporter) Run() {
+	for res := range r.results {
+		r.lock.Lock()
+
+		r.latencies = append(r.latencies, res.duration.Milliseconds())
+
+		if res.statusCode != nil {
+			r.statusCodes[*res.statusCode]++
+		}
+
+		if res.err != nil {
+			if len(r.errors) < 1_000_000 {
+				r.errors[res.err.Error()]++
+			} else {
+				// we don't want to store too many errors in memory
+				r.errors["hidden"]++
+			}
+		} else {
+			r.errors["nil"]++
+		}
+
+		r.lock.Unlock()
+	}
+
+	r.doneCh <- true
+}
+
+func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	nrReq := int64(len(r.latencies))
+	if nrReq == 0 {
+		fmt.Println("No requests were made")
+		return
+	}
+
+	// we need to sort the latencies slice to calculate the percentiles
+	sort.Slice(r.latencies, func(i, j int) bool {
+		return r.latencies[i] < r.latencies[j]
+	})
+
+	var totalLatency int64 = 0
+	for _, latency := range r.latencies {
+		totalLatency += latency
+	}
+
+	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
+	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
+	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
+	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
+	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
+	fmt.Fprintf(w, "- Latency distribution:\n")
+	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
+	for _, p := range percentiles {
+		idx := int64(p * float64(nrReq))
+		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
+	}
+
+	// create a simple histogram with 10 buckets spanning the range of latency
+	// into equal ranges
+	//
+	nrBucket := 10
+	buckets := make([]Bucket, nrBucket)
+	latencyRange := r.latencies[len(r.latencies)-1]
+	bucketRange := latencyRange / int64(nrBucket)
+
+	// mark the end of each bucket
+	for i := 0; i < nrBucket; i++ {
+		buckets[i].start = int64(i) * bucketRange
+		buckets[i].end = buckets[i].start + bucketRange
+		// extend the last bucked by any remaning range caused by the integer division
+		if i == nrBucket-1 {
+			buckets[i].end = latencyRange
+		}
+	}
+
+	// count the number of requests in each bucket
+	currBucket := 0
+	for i := 0; i < len(r.latencies); {
+		if r.latencies[i] <= buckets[currBucket].end {
+			buckets[currBucket].cnt++
+			i++
+		} else {
+			currBucket++
+		}
+	}
+
+	// print the histogram using a tabwriter which will align the columns nicely
+	fmt.Fprintf(w, "- Histogram:\n")
+	const padding = 2
+	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
+	for i := 0; i < nrBucket; i++ {
+		ratio := float64(buckets[i].cnt) / float64(nrReq)
+		bars := strings.Repeat("#", int(ratio*100))
+		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
+	}
+	tabWriter.Flush() //nolint:errcheck
+
+	fmt.Fprintf(w, "- Status codes:\n")
+	for code, cnt := range r.statusCodes {
+		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
+	}
+
+	// print the 10 most occurring errors (in case error values are not unique)
+	//
+	type kv struct {
+		err string
+		cnt int
+	}
+	var sortedErrors []kv
+	for err, cnt := range r.errors {
+		sortedErrors = append(sortedErrors, kv{err, cnt})
+	}
+	sort.Slice(sortedErrors, func(i, j int) bool {
+		return sortedErrors[i].cnt > sortedErrors[j].cnt
+	})
+	fmt.Fprintf(w, "- Errors (top 10):\n")
+	for i, se := range sortedErrors {
+		if i > 10 {
+			break
+		}
+		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
+	}
+}
+
+type Bucket struct {
+	start int64
+	// the end value of the bucket
+	end int64
+	// how many entries are in the bucket
+	cnt int
+}

cmd/lotus-bench/rpc.go

@@ -9,11 +9,9 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
-	"text/tabwriter"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -243,13 +241,6 @@ type RPCMethod struct {
 	reporter *Reporter
 }
 
-// result is the result of a single rpc method request.
-type result struct {
-	err        error
-	statusCode *int
-	duration   time.Duration
-}
-
 func (rpc *RPCMethod) Run() error {
 	client := &http.Client{
 		Timeout: 0,
@@ -411,166 +402,3 @@ func (rpc *RPCMethod) Report() {
 	fmt.Fprintf(rpc.w, "  - qps: %d\n", rpc.qps)
 	rpc.reporter.Print(total, rpc.w)
 }
-
-// Reporter reads the results from the workers through the results channel and aggregates the results.
-type Reporter struct {
-	// write the report to this writer
-	w io.Writer
-	// the reporter read the results from this channel
-	results chan *result
-	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
-	doneCh chan bool
-
-	// lock protect the following fields during critical sections (if --watch was specified)
-	lock sync.Mutex
-	// the latencies of all requests
-	latencies []int64
-	// the number of requests that returned each status code
-	statusCodes map[int]int
-	// the number of errors that occurred
-	errors map[string]int
-}
-
-func NewReporter(results chan *result, w io.Writer) *Reporter {
-	return &Reporter{
-		w:           w,
-		results:     results,
-		doneCh:      make(chan bool, 1),
-		statusCodes: make(map[int]int),
-		errors:      make(map[string]int),
-	}
-}
-
-func (r *Reporter) Run() {
-	for res := range r.results {
-		r.lock.Lock()
-
-		r.latencies = append(r.latencies, res.duration.Milliseconds())
-
-		if res.statusCode != nil {
-			r.statusCodes[*res.statusCode]++
-		}
-
-		if res.err != nil {
-			if len(r.errors) < 1_000_000 {
-				r.errors[res.err.Error()]++
-			} else {
-				// we don't want to store too many errors in memory
-				r.errors["hidden"]++
-			}
-		} else {
-			r.errors["nil"]++
-		}
-
-		r.lock.Unlock()
-	}
-
-	r.doneCh <- true
-}
-
-func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
-	r.lock.Lock()
-	defer r.lock.Unlock()
-
-	nrReq := int64(len(r.latencies))
-	if nrReq == 0 {
-		fmt.Println("No requests were made")
-		return
-	}
-
-	// we need to sort the latencies slice to calculate the percentiles
-	sort.Slice(r.latencies, func(i, j int) bool {
-		return r.latencies[i] < r.latencies[j]
-	})
-
-	var totalLatency int64 = 0
-	for _, latency := range r.latencies {
-		totalLatency += latency
-	}
-
-	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
-	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
-	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
-	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
-	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
-	fmt.Fprintf(w, "- Latency distribution:\n")
-	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
-	for _, p := range percentiles {
-		idx := int64(p * float64(nrReq))
-		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
-	}
-
-	// create a simple histogram with 10 buckets spanning the range of latency
-	// into equal ranges
-	//
-	nrBucket := 10
-	buckets := make([]Bucket, nrBucket)
-	latencyRange := r.latencies[len(r.latencies)-1]
-	bucketRange := latencyRange / int64(nrBucket)
-
-	// mark the end of each bucket
-	for i := 0; i < nrBucket; i++ {
-		buckets[i].start = int64(i) * bucketRange
-		buckets[i].end = buckets[i].start + bucketRange
-		// extend the last bucked by any remaning range caused by the integer division
-		if i == nrBucket-1 {
-			buckets[i].end = latencyRange
-		}
-	}
-
-	// count the number of requests in each bucket
-	currBucket := 0
-	for i := 0; i < len(r.latencies); {
-		if r.latencies[i] <= buckets[currBucket].end {
-			buckets[currBucket].cnt++
-			i++
-		} else {
-			currBucket++
-		}
-	}
-
-	// print the histogram using a tabwriter which will align the columns nicely
-	fmt.Fprintf(w, "- Histogram:\n")
-	const padding = 2
-	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
-	for i := 0; i < nrBucket; i++ {
-		ratio := float64(buckets[i].cnt) / float64(nrReq)
-		bars := strings.Repeat("#", int(ratio*100))
-		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
-	}
-	tabWriter.Flush() //nolint:errcheck
-
-	fmt.Fprintf(w, "- Status codes:\n")
-	for code, cnt := range r.statusCodes {
-		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
-	}
-
-	// print the 10 most occurring errors (in case error values are not unique)
-	//
-	type kv struct {
-		err string
-		cnt int
-	}
-	var sortedErrors []kv
-	for err, cnt := range r.errors {
-		sortedErrors = append(sortedErrors, kv{err, cnt})
-	}
-	sort.Slice(sortedErrors, func(i, j int) bool {
-		return sortedErrors[i].cnt > sortedErrors[j].cnt
-	})
-	fmt.Fprintf(w, "- Errors (top 10):\n")
-	for i, se := range sortedErrors {
-		if i > 10 {
-			break
-		}
-		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
-	}
-}
-
-type Bucket struct {
-	start int64
-	// the end value of the bucket
-	end int64
-	// how many entries are in the bucket
-	cnt int
-}

cmd/lotus-bench/simple.go

@@ -266,7 +266,10 @@ var simplePreCommit1 = &cli.Command{
 			ProofType: spt(sectorSize, cctx.Bool("synthetic")),
 		}
 
-		var ticket [32]byte // all zero
+		ticket := [32]byte{}
+		for i := range ticket {
+			ticket[i] = 1
+		}
 
 		pieces, err := ParsePieceInfos(cctx, 3)
 		if err != nil {
@@ -305,7 +308,36 @@ var simplePreCommit2 = &cli.Command{
 			Name:  "synthetic",
 			Usage: "generate synthetic PoRep proofs",
 		},
+		&cli.StringFlag{
+			Name:  "external-pc2",
+			Usage: "command for computing PC2 externally",
+		},
 	},
+	Description: `Compute PreCommit2 inputs and seal a sector.
+
+--external-pc2 can be used to compute the PreCommit2 inputs externally.
+The flag behaves similarly to the related lotus-worker flag, using it in
+lotus-bench may be useful for testing if the external PreCommit2 command is
+invoked correctly.
+
+The command will be called with a number of environment variables set:
+* EXTSEAL_PC2_SECTOR_NUM: the sector number
+* EXTSEAL_PC2_SECTOR_MINER: the miner id
+* EXTSEAL_PC2_PROOF_TYPE: the proof type
+* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
+* EXTSEAL_PC2_CACHE: the path to the cache directory
+* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
+* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
+
+The command is expected to:
+* Create cache sc-02-data-tree-r* files
+* Create cache sc-02-data-tree-c* files
+* Create cache p_aux / t_aux files
+* Transform the sealed file in place
+
+Example invocation of lotus-bench as external executor:
+'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
+`,
 	ArgsUsage: "[sealed] [cache] [pc1 out]",
 	Action: func(cctx *cli.Context) error {
 		ctx := cctx.Context
@@ -330,7 +362,18 @@ var simplePreCommit2 = &cli.Command{
 			storiface.FTSealed: cctx.Args().Get(0),
 			storiface.FTCache:  cctx.Args().Get(1),
 		}
-		sealer, err := ffiwrapper.New(pp)
+
+		var opts []ffiwrapper.FFIWrapperOpt
+
+		if cctx.IsSet("external-pc2") {
+			extSeal := ffiwrapper.ExternalSealer{
+				PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")),
+			}
+
+			opts = append(opts, ffiwrapper.WithExternalSealCalls(extSeal))
+		}
+
+		sealer, err := ffiwrapper.New(pp, opts...)
 		if err != nil {
 			return err
 		}
@@ -420,7 +463,12 @@ var simpleCommit1 = &cli.Command{
 
 		start := time.Now()
 
-		var ticket, seed [32]byte // all zero
+		ticket := [32]byte{}
+		seed := [32]byte{}
+		for i := range ticket {
+			ticket[i] = 1
+			seed[i] = 1
+		}
 
 		commd, err := cid.Parse(cctx.Args().Get(2))
 		if err != nil {
@@ -650,6 +698,10 @@ var simpleWinningPost = &cli.Command{
 			Usage: "pass miner address (only necessary if using existing sectorbuilder)",
 			Value: "t01000",
 		},
+		&cli.BoolFlag{
+			Name:  "show-inputs",
+			Usage: "output inputs for winning post generation",
+		},
 	},
 	ArgsUsage: "[sealed] [cache] [comm R] [sector num]",
 	Action: func(cctx *cli.Context) error {
@@ -720,6 +772,17 @@ var simpleWinningPost = &cli.Command{
 		fmt.Printf("Vanilla %s (%s)\n", challenge.Sub(start), bps(sectorSize, 1, challenge.Sub(start)))
 		fmt.Printf("Proof %s (%s)\n", end.Sub(challenge), bps(sectorSize, 1, end.Sub(challenge)))
 		fmt.Println(base64.StdEncoding.EncodeToString(proof[0].ProofBytes))
+
+		if cctx.Bool("show-inputs") {
+			fmt.Println("GenerateWinningPoStWithVanilla info:")
+
+			fmt.Printf(" wpt: %d\n", wpt)
+			fmt.Printf(" mid: %d\n", mid)
+			fmt.Printf(" rand: %x\n", rand)
+			fmt.Printf(" vp: %x\n", vp)
+			fmt.Printf(" proof: %x\n", proof)
+		}
+
 		return nil
 	},
 }

cmd/lotus-gateway/main.go

@@ -143,7 +143,7 @@ var runCmd = &cli.Command{
 		},
 		&cli.DurationFlag{
 			Name:  "rate-limit-timeout",
-			Usage: "the maximum time to wait for the rate limter before returning an error to clients",
+			Usage: "the maximum time to wait for the rate limiter before returning an error to clients",
 			Value: gateway.DefaultRateLimitTimeout,
 		},
 		&cli.Int64Flag{

cmd/lotus-miner/sectors.go

@@ -2566,6 +2566,8 @@ var sectorsUnsealCmd = &cli.Command{
 			return xerrors.Errorf("could not parse sector number: %w", err)
 		}
 
+		fmt.Printf("Unsealing sector %d\n", sectorNum)
+
 		return minerAPI.SectorUnseal(ctx, abi.SectorNumber(sectorNum))
 	},
 }

cmd/lotus-shed/genesis-verify.go

@@ -62,7 +62,7 @@ var genesisVerifyCmd = &cli.Command{
 			return xerrors.Errorf("opening the car file: %w", err)
 		}
 
-		ts, err := cs.Import(cctx.Context, f)
+		ts, _, err := cs.Import(cctx.Context, f)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/mpool.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/urfave/cli/v2"
 
@@ -43,10 +44,20 @@ var minerSelectMsgsCmd = &cli.Command{
 			return err
 		}
 
+		// Get the size of the mempool
+		pendingMsgs, err := api.MpoolPending(ctx, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+		mpoolSize := len(pendingMsgs)
+
+		// Measure the time taken by MpoolSelect
+		startTime := time.Now()
 		msgs, err := api.MpoolSelect(ctx, head.Key(), cctx.Float64("ticket-quality"))
 		if err != nil {
 			return err
 		}
+		duration := time.Since(startTime)
 
 		var totalGas int64
 		for i, f := range msgs {
@@ -64,6 +75,9 @@ var minerSelectMsgsCmd = &cli.Command{
 			totalGas += f.Message.GasLimit
 		}
 
+		// Log the duration, size of the mempool, selected messages and total gas limit of selected messages
+		fmt.Printf("Message selection took %s\n", duration)
+		fmt.Printf("Size of the mempool: %d\n", mpoolSize)
 		fmt.Println("selected messages: ", len(msgs))
 		fmt.Printf("total gas limit of selected messages: %d / %d (%0.2f%%)\n", totalGas, build.BlockGasLimit, 100*float64(totalGas)/float64(build.BlockGasLimit))
 		return nil

cmd/lotus-shed/pruning.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 
-	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -144,13 +144,6 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}()
 
-		// After migrating to native blockstores, this has been made
-		// database-specific.
-		badgbs, ok := bs.(*badgerbs.Blockstore)
-		if !ok {
-			return fmt.Errorf("only badger blockstores are supported")
-		}
-
 		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
@@ -160,8 +153,12 @@ var stateTreePruneCmd = &cli.Command{
 		const DiscardRatio = 0.2
 		if cctx.Bool("only-ds-gc") {
 			fmt.Println("running datastore gc....")
+			gbs, ok := bs.(blockstore.BlockstoreGCOnce)
+			if !ok {
+				return xerrors.Errorf("blockstore %T does not support GC", bs)
+			}
 			for i := 0; i < cctx.Int("gc-count"); i++ {
-				if err := badgbs.DB().RunValueLogGC(DiscardRatio); err != nil {
+				if err := gbs.GCOnce(ctx, blockstore.WithThreshold(DiscardRatio)); err != nil {
 					return xerrors.Errorf("datastore GC failed: %w", err)
 				}
 			}
@@ -208,13 +205,6 @@ var stateTreePruneCmd = &cli.Command{
 			return nil
 		}
 
-		b := badgbs.DB().NewWriteBatch()
-		defer b.Cancel()
-
-		markForRemoval := func(c cid.Cid) error {
-			return b.Delete(badgbs.StorageKey(nil, c))
-		}
-
 		keys, err := bs.AllKeysChan(context.Background())
 		if err != nil {
 			return xerrors.Errorf("failed to query blockstore: %w", err)
@@ -225,12 +215,12 @@ var stateTreePruneCmd = &cli.Command{
 		var deleteCount int
 		var goodHits int
 		for k := range keys {
-			if goodSet.HasRaw(k.Bytes()) {
+			if goodSet.Has(k) {
 				goodHits++
 				continue
 			}
 
-			if err := markForRemoval(k); err != nil {
+			if err := bs.DeleteBlock(ctx, k); err != nil {
 				return fmt.Errorf("failed to remove cid %s: %w", k, err)
 			}
 
@@ -243,13 +233,15 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}
 
-		if err := b.Flush(); err != nil {
-			return xerrors.Errorf("failed to flush final batch delete: %w", err)
+		fmt.Println("running datastore gc....")
+		gbs, ok := bs.(blockstore.BlockstoreGCOnce)
+		if !ok {
+			fmt.Println("gc not supported...")
+			return nil
 		}
 
-		fmt.Println("running datastore gc....")
 		for i := 0; i < cctx.Int("gc-count"); i++ {
-			if err := badgbs.DB().RunValueLogGC(DiscardRatio); err != nil {
+			if err := gbs.GCOnce(ctx, blockstore.WithThreshold(DiscardRatio)); err != nil {
 				return xerrors.Errorf("datastore GC failed: %w", err)
 			}
 		}

cmd/lotus-worker/main.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -38,6 +39,7 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer"
+	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
 	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
@@ -283,7 +285,36 @@ var runCmd = &cli.Command{
 			Value:       true,
 			DefaultText: "inherits --addpiece",
 		},
+		&cli.StringFlag{
+			Name:  "external-pc2",
+			Usage: "command for computing PC2 externally",
+		},
 	},
+	Description: `Run lotus-worker.
+
+--external-pc2 can be used to compute the PreCommit2 inputs externally.
+The flag behaves similarly to the related lotus-worker flag, using it in
+lotus-bench may be useful for testing if the external PreCommit2 command is
+invoked correctly.
+
+The command will be called with a number of environment variables set:
+* EXTSEAL_PC2_SECTOR_NUM: the sector number
+* EXTSEAL_PC2_SECTOR_MINER: the miner id
+* EXTSEAL_PC2_PROOF_TYPE: the proof type
+* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
+* EXTSEAL_PC2_CACHE: the path to the cache directory
+* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
+* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
+
+The command is expected to:
+* Create cache sc-02-data-tree-r* files
+* Create cache sc-02-data-tree-c* files
+* Create cache p_aux / t_aux files
+* Transform the sealed file in place
+
+Example invocation of lotus-bench as external executor:
+'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
+`,
 	Before: func(cctx *cli.Context) error {
 		if cctx.IsSet("address") {
 			log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'")
@@ -348,6 +379,18 @@ var runCmd = &cli.Command{
 		// Connect to storage-miner
 		ctx := lcli.ReqContext(cctx)
 
+		// Create a new context with cancel function
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		// Listen for interrupt signals
+		go func() {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, os.Interrupt)
+			<-c
+			cancel()
+		}()
+
 		var nodeApi api.StorageMiner
 		var closer func()
 		for {
@@ -359,14 +402,13 @@ var runCmd = &cli.Command{
 				}
 			}
 			fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err)
-			time.Sleep(time.Second)
-			continue
+			select {
+			case <-ctx.Done():
+				return xerrors.New("Interrupted by user")
+			case <-time.After(time.Second):
+			}
 		}
-
 		defer closer()
-		ctx, cancel := context.WithCancel(ctx)
-		defer cancel()
-
 		// Register all metric views
 		if err := view.Register(
 			metrics.DefaultViews...,
@@ -611,18 +653,32 @@ var runCmd = &cli.Command{
 			fh.ServeHTTP(w, r)
 		}
 
+		// Parse ffi executor flags
+
+		var ffiOpts []ffiwrapper.FFIWrapperOpt
+
+		if cctx.IsSet("external-pc2") {
+			extSeal := ffiwrapper.ExternalSealer{
+				PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")),
+			}
+
+			ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal))
+		}
+
 		// Create / expose the worker
 
 		wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix))
 
 		workerApi := &sealworker.Worker{
-			LocalWorker: sealer.NewLocalWorker(sealer.WorkerConfig{
-				TaskTypes:                 taskTypes,
-				NoSwap:                    cctx.Bool("no-swap"),
-				MaxParallelChallengeReads: cctx.Int("post-parallel-reads"),
-				ChallengeReadTimeout:      cctx.Duration("post-read-timeout"),
-				Name:                      cctx.String("name"),
-			}, remote, localStore, nodeApi, nodeApi, wsts),
+			LocalWorker: sealer.NewLocalWorkerWithExecutor(
+				sealer.FFIExec(ffiOpts...),
+				sealer.WorkerConfig{
+					TaskTypes:                 taskTypes,
+					NoSwap:                    cctx.Bool("no-swap"),
+					MaxParallelChallengeReads: cctx.Int("post-parallel-reads"),
+					ChallengeReadTimeout:      cctx.Duration("post-read-timeout"),
+					Name:                      cctx.String("name"),
+				}, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts),
 			LocalStore: localStore,
 			Storage:    lr,
 		}

cmd/lotus/daemon.go

@@ -269,7 +269,37 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		if cctx.Bool("remove-existing-chain") {
+		chainfile := cctx.String("import-chain")
+		snapshot := cctx.String("import-snapshot")
+		willImportChain := false
+		if chainfile != "" || snapshot != "" {
+			if chainfile != "" && snapshot != "" {
+				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
+			}
+			willImportChain = true
+		}
+
+		willRemoveChain := cctx.Bool("remove-existing-chain")
+		if willImportChain && !willRemoveChain {
+			// Confirm with the user about the intention to remove chain data.
+			reader := bufio.NewReader(os.Stdin)
+			fmt.Print("Importing chain or snapshot will by default delete existing local chain data. Do you want to proceed and delete? (yes/no): ")
+			userInput, err := reader.ReadString('\n')
+			if err != nil {
+				return xerrors.Errorf("reading user input: %w", err)
+			}
+			userInput = strings.ToLower(strings.TrimSpace(userInput))
+
+			if userInput == "yes" {
+				willRemoveChain = true
+			} else if userInput == "no" {
+				willRemoveChain = false
+			} else {
+				return fmt.Errorf("invalid input. please answer with 'yes' or 'no'")
+			}
+		}
+
+		if willRemoveChain {
 			lr, err := repo.NewFS(cctx.String("repo"))
 			if err != nil {
 				return xerrors.Errorf("error opening fs repo: %w", err)
@@ -289,12 +319,7 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		chainfile := cctx.String("import-chain")
-		snapshot := cctx.String("import-snapshot")
-		if chainfile != "" || snapshot != "" {
-			if chainfile != "" && snapshot != "" {
-				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
-			}
+		if willImportChain {
 			var issnapshot bool
 			if chainfile == "" {
 				chainfile = snapshot
@@ -540,7 +565,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 	}
 
 	bar.Start()
-	ts, err := cst.Import(ctx, ir)
+	ts, gen, err := cst.Import(ctx, ir)
 	bar.Finish()
 
 	if err != nil {
@@ -551,18 +576,14 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 		return xerrors.Errorf("flushing validation cache failed: %w", err)
 	}
 
-	gb, err := cst.GetTipsetByHeight(ctx, 0, ts, true)
-	if err != nil {
-		return err
-	}
-
-	err = cst.SetGenesis(ctx, gb.Blocks()[0])
+	log.Infof("setting genesis")
+	err = cst.SetGenesis(ctx, gen)
 	if err != nil {
 		return err
 	}
 
 	if !snapshot {
-		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), gb.MinTimestamp(), nil)
+		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), gen.Timestamp, nil)
 		if err != nil {
 			return xerrors.Errorf("failed to construct beacon schedule: %w", err)
 		}

documentation/en/api-v0-methods.md

@@ -2289,7 +2289,8 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   },
   [
     {
@@ -2766,7 +2767,8 @@ Inputs:
   ],
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   }
 ]
 ```
@@ -3025,7 +3027,8 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   }
 ]
 ```

documentation/en/api-v1-unstable-methods.md

@@ -2402,14 +2402,7 @@ Perms: read
 Inputs:
 ```json
 [
-  {
-    "from": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
-    "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
-    "gas": "0x5",
-    "gasPrice": "0x0",
-    "value": "0x0",
-    "data": "0x07"
-  }
+  "Bw=="
 ]
 ```
 
@@ -3357,7 +3350,8 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   },
   [
     {
@@ -3834,7 +3828,8 @@ Inputs:
   ],
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   }
 ]
 ```
@@ -4226,7 +4221,8 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707"
+    "MsgUuid": "07070707-0707-0707-0707-070707070707",
+    "MaximizeFeeCap": true
   }
 ]
 ```

documentation/en/block-validation.md

@@ -104,9 +104,9 @@ domain separation tag.
 ### Winning PoSt proof
 
 Draw randomness for current epoch with `WinningPoSt` domain separation tag.
-Get list of sectors challanged in this epoch for this miner, based on the randomness drawn. 
+Get list of sectors challenged in this epoch for this miner, based on the randomness drawn. 
 
-`V`: Use filecoin proofs system to verify that miner prooved access to sealed versions of these sectors.
+`V`: Use filecoin proofs system to verify that miner proved access to sealed versions of these sectors.
 
 ## `(*StateManager).TipSetState()`
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1
+   1.25.1-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1
+   1.25.1-dev
 
 COMMANDS:
    run        Start lotus worker
@@ -34,6 +34,33 @@ NAME:
 USAGE:
    lotus-worker run [command options] [arguments...]
 
+DESCRIPTION:
+   Run lotus-worker.
+
+   --external-pc2 can be used to compute the PreCommit2 inputs externally.
+   The flag behaves similarly to the related lotus-worker flag, using it in
+   lotus-bench may be useful for testing if the external PreCommit2 command is
+   invoked correctly.
+
+   The command will be called with a number of environment variables set:
+   * EXTSEAL_PC2_SECTOR_NUM: the sector number
+   * EXTSEAL_PC2_SECTOR_MINER: the miner id
+   * EXTSEAL_PC2_PROOF_TYPE: the proof type
+   * EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
+   * EXTSEAL_PC2_CACHE: the path to the cache directory
+   * EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
+   * EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
+
+   The command is expected to:
+   * Create cache sc-02-data-tree-r* files
+   * Create cache sc-02-data-tree-c* files
+   * Create cache p_aux / t_aux files
+   * Transform the sealed file in place
+
+   Example invocation of lotus-bench as external executor:
+   './lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
+
+
 OPTIONS:
    --listen value                host address and port the worker api will listen on (default: "0.0.0.0:3456") [$LOTUS_WORKER_LISTEN]
    --no-local-storage            don't use storageminer repo for sector storage (default: false) [$LOTUS_WORKER_NO_LOCAL_STORAGE]
@@ -57,6 +84,7 @@ OPTIONS:
    --timeout value               used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function (default: "30m") [$LOTUS_WORKER_TIMEOUT]
    --http-server-timeout value   (default: "30s")
    --data-cid                    Run the data-cid task. true|false (default: inherits --addpiece)
+   --external-pc2 value          command for computing PC2 externally
    --help, -h                    show help
 ```
 

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1
+   1.25.1-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process

documentation/en/default-lotus-config.toml

@@ -195,7 +195,7 @@
 
   [Chainstore.Splitstore]
     # ColdStoreType specifies the type of the coldstore.
-    # It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
+    # It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state..
     #
     # type: string
     # env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE

documentation/en/default-lotus-miner-config.toml

@@ -778,6 +778,10 @@
   # env var: LOTUS_FEES_MAXMARKETBALANCEADDFEE
   #MaxMarketBalanceAddFee = "0.007 FIL"
 
+  # type: bool
+  # env var: LOTUS_FEES_MAXIMIZEWINDOWPOSTFEECAP
+  #MaximizeWindowPoStFeeCap = true
+
   [Fees.MaxPreCommitBatchGasFee]
     # type: types.FIL
     # env var: LOTUS_FEES_MAXPRECOMMITBATCHGASFEE_BASE

documentation/misc/RELEASE_ISSUE_TEMPLATE.md

@@ -14,11 +14,13 @@
 **Note for whoever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases.  There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end.
 
 First steps:
-  - [ ] FFI: Fork a new branch (`release/lotus-vX.Y.Z`) from the filecoin-ffi `master` branch
-  - [ ] FFI: Tag the head of `release/lotus-vX.Y.Z` as `vX.Y.Z-pre1`
+  - [ ] Set shell variables vX.Y.Z `export X=1 && export Y=24 && export Z=3`
+  - [ ] FFI: Fork a new branch (`release/lotus-vX.Y.Z`) from the filecoin-ffi `master` branch: `git checkout master && git checkout -b release/lotus-v$X.$Y.$Z`
+  - [ ] FFI: Tag the head of `release/lotus-vX.Y.Z` as `vX.Y.Z-pre1`: `git tag -a v$X.$Y.$Z-pre1 -m"release"`
   - [ ] Open and land a PR in lotus `master` that updates the FFI dependency to `vX.Y.Z-pre1` as cut in the previous step
   - [ ] Fork a new branch (`release/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage.
-  - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release). Run make gen and make docsgen-cli before committing changes
+  - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release).
+  - [ ] Run `make gen && make docsgen-cli` before committing changes
     
 **Prepping an RC**:
 
@@ -26,6 +28,7 @@ First steps:
 - [ ] run `make gen && make docsgen-cli`
 - [ ] Generate changelog using the script at scripts/mkreleaselog
 - [ ] Add contents of generated text to lotus/CHANGELOG.md in addition to other details
+- [ ] Commit using PR
 - [ ] tag commit with `vX.Y.Z-rcN`
 - [ ] cut a pre-release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=true)
 

gateway/node.go

@@ -131,7 +131,7 @@ type TargetAPI interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)

gateway/proxy_eth.go

@@ -353,13 +353,19 @@ func (gw *Node) EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt
 	return gw.target.EthMaxPriorityFeePerGas(ctx)
 }
 
-func (gw *Node) EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error) {
+func (gw *Node) EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+	// validate params
+	_, err := jsonrpc.DecodeParams[ethtypes.EthEstimateGasParams](p)
+	if err != nil {
+		return ethtypes.EthUint64(0), xerrors.Errorf("decoding params: %w", err)
+	}
+
 	if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
 		return 0, err
 	}
 
 	// todo limit gas? to what?
-	return gw.target.EthEstimateGas(ctx, tx)
+	return gw.target.EthEstimateGas(ctx, p)
 }
 
 func (gw *Node) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) {

gen/main.go

@@ -92,7 +92,7 @@ func main() {
 	err = gen.WriteTupleEncodersToFile("./chain/exchange/cbor_gen.go", "exchange",
 		exchange.Request{},
 		exchange.Response{},
-		exchange.CompactedMessagesCBOR{},
+		exchange.CompactedMessages{},
 		exchange.BSTipSet{},
 	)
 	if err != nil {

go.mod

@@ -107,7 +107,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
-	github.com/libp2p/go-libp2p v0.30.0
+	github.com/libp2p/go-libp2p v0.31.0
 	github.com/libp2p/go-libp2p-consensus v0.0.1
 	github.com/libp2p/go-libp2p-gorpc v0.5.0
 	github.com/libp2p/go-libp2p-kad-dht v0.24.0
@@ -136,6 +136,7 @@ require (
 	github.com/raulk/go-watchdog v1.3.0
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7
 	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
@@ -297,7 +298,7 @@ require (
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.37.6 // indirect
+	github.com/quic-go/quic-go v0.38.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
@@ -338,3 +339,5 @@ require (
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
+
+replace github.com/triplewz/poseidon => github.com/magik6k/poseidon v0.0.0-neptune