add index to event.emitter_addr

* exclude reverted events from results returned by eth_getLogs

* unit test

* update CHANGELOG.md

.circleci/config.yml

@@ -996,20 +996,23 @@ workflows:
             - build
           suite: utest-unit-cli
           target: "./cli/... ./cmd/... ./api/..."
-          executor: golang-2xl
           get-params: true
+          executor: golang-2xl
       - test:
           name: test-unit-node
           requires:
             - build
           suite: utest-unit-node
           target: "./node/..."
+          
+          
       - test:
           name: test-unit-rest
           requires:
             - build
           suite: utest-unit-rest
           target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
+          
           executor: golang-2xl
       - test:
           name: test-unit-storage
@@ -1017,7 +1020,8 @@ workflows:
             - build
           suite: utest-unit-storage
           target: "./storage/... ./extern/..."
-          get-params: true
+          
+          
       - test:
           go-test-flags: "-run=TestMulticoreSDR"
           requires:

.circleci/template.yml

@@ -557,16 +557,9 @@ workflows:
             - build
           suite: utest-[[ $suite ]]
           target: "[[ $pkgs ]]"
-          [[- if eq $suite "unit-storage"]]
-          get-params: true
-          [[- end -]]
-          [[- if eq $suite "unit-cli"]]
-          executor: golang-2xl
-          get-params: true
-          [[- end -]]
-          [[- if eq $suite "unit-rest"]]
-          executor: golang-2xl
-          [[- end -]]
+          [[if eq $suite "unit-cli"]]get-params: true[[end]]
+          [[if eq $suite "unit-cli"]]executor: golang-2xl[[end]]
+          [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]]
       [[- end]]
       - test:
           go-test-flags: "-run=TestMulticoreSDR"

.gitignore

@@ -41,7 +41,6 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
-.vscode
 scratchpad
 
 build/builtin-actors/v*

.golangci.yml

@@ -14,7 +14,6 @@ linters:
     - varcheck
     - deadcode
     - scopelint
-    - unused
 
 # We don't want to skip builtin/
 skip-dirs-use-default: false

CHANGELOG.md

@@ -1,21 +1,14 @@
 # Lotus changelog
 
-# UNRELEASED
-- chore: Auto remove local chain data when importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
-- feat: metric: export Mpool message count ([filecoin-project/lotus#11361](https://github.com/filecoin-project/lotus/pull/11361))
-- feat: sealing: load SectorsSummary from sealing SectorStats instead of calling API each time ([filecoin-project/lotus#11353](https://github.com/filecoin-project/lotus/pull/11353))
-- fix: miner info: Show correct sector state counts ([filecoin-project/lotus#11456](https://github.com/filecoin-project/lotus/pull/11456))
-- feat: add support for specifying block number when calling `eth_estimateGas` ([filecoin-project/lotus#11462](https://github.com/filecoin-project/lotus/pull/11462)).
+# v1.25.1 / 2023-12-09
+
+This is a **highly recommended PATCH RELEASE.**  The patch release fixes the issue were node operators trying to catch up sync were unable to sync large message blocks/epochs due to an increased number of messages on the network.
+
+This patch release allows for up to 10k messages per block. Additionally, it introduces a limit on the amount of data that can be read at once, ensuring the system can handle worst-case scenarios.
 
 ## Improvements
-- fix: Add time slicing to splitstore purging step during compaction to reduce lock congestion [filecoin-project/lotus#11269](https://github.com/filecoin-project/lotus/pull/11269)
-- feat: Added instructions on how to setup Prometheus/Grafana for monitoring a local Lotus node [filecoin-project/lotus#11276](https://github.com/filecoin-project/lotus/pull/11276)
-- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
-- fix: The Ethereum API will now use the correct state-tree when resolving "native" addresses into masked ID addresses. Additionally, pending messages from native account types won't be visible in the Ethereum API because there is no "correct" state-tree to pick in this case. However, pending _Ethereum_ transactions and native messages that have landed on-chain will still be visible through the Ethereum API.
-- feat: Unambiguously translate native messages to Ethereum transactions by:
-  - Detecting native messages that "look" like Ethereum transactions (creating smart contracts, invoking a smart contract, etc.), and decoding them as such.
-  - Otherwise, ABI-encoding the inputs as if they were calls to a `handle_filecoin_method` Solidity method.
-- fix: ensure that the Ethereum API never returns "empty" addresses for native messages. When a "to" address cannot be resolved to a 0x-style address, it will be re-written to `0xff0000000000000000000000ffffffffffffffff`. This can only happen when the native transaction _reverted_ (failing to create an account at the specified "to" address).
+- fix: exchange: allow up to 10k messages per block ([filecoin-project/lotus#11506](https://github.com/filecoin-project/lotus/pull/11506))
+
 
 # v 1.25.0 / 2023-11-22
 
@@ -101,6 +94,7 @@ Lotus-workers can now be built to leverage the SupraSeal C2 sealing optimization
 - fix(client): single-root error message ([filecoin-project/lotus#11214](https://github.com/filecoin-project/lotus/pull/11214))
 - fix: worker: Convert `DC_[SectorSize]_[ResourceRestriction]` if set ([filecoin-project/lotus#11224](https://github.com/filecoin-project/lotus/pull/11224))
 - chore: backport #11338 onto release/v1.25.0 ([filecoin-project/lotus#11350](https://github.com/filecoin-project/lotus/pull/11350))
+- fix: Exclude reverted events in `eth_getLogs` results [filecoin-project/lotus#11318](https://github.com/filecoin-project/lotus/pull/11318)
 
 ## Dependencies
 - deps: update go-libp2p to v0.28.1 ([filecoin-project/lotus#10998](https://github.com/filecoin-project/lotus/pull/10998))
@@ -262,7 +256,7 @@ This feature release requires a **minimum Go version of v1.19.12 or higher to su
   - feat: sealing: flag to run data_cid untied from addpiece ([filecoin-project/lotus#10797](https://github.com/filecoin-project/lotus/pull/10797))
   - feat: Lotus Gateway: add MpoolPending, ChainGetBlock and MinerGetBaseInfo ([filecoin-project/lotus#10929](https://github.com/filecoin-project/lotus/pull/10929))
 
-## Improvements && Bug Fixes
+## Improvements && Bug Fixe
   - chore: update ffi & fvm ([filecoin-project/lotus#11040](https://github.com/filecoin-project/lotus/pull/11040))
   - feat: Make sure we don't store duplidate actor events caused to reorgs in events.db ([filecoin-project/lotus#11015](https://github.com/filecoin-project/lotus/pull/11015))
   - sealing: Use only non-assigned deals when selecting snap sectors ([filecoin-project/lotus#11002](https://github.com/filecoin-project/lotus/pull/11002))

LOTUS_RELEASE_FLOW.md

@@ -73,7 +73,7 @@ All releases under an odd minor version number indicate **feature releases**. Th
 
 Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes.  There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
 
-We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the [Release Cycle section](#release-cycle).
+We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the Release Cycle section (TODO: Link).
 
 ### Examples Scenarios
 

README.md

@@ -71,10 +71,10 @@ For other distributions you can find the required dependencies [here.](https://l
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.20.10 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.19.12 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.20.10.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.19.12.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**
@@ -133,8 +133,6 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
 6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
 
-7. (Optional) Follow the [Setting Up Prometheus and Grafana](https://github.com/filecoin-project/lotus/tree/master/metrics/README.md) guide for detailed instructions on setting up a working monitoring system running against a local running lotus node.
-
 ## License
 
 Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE)

api/api_full.go

@@ -824,7 +824,7 @@ type FullNode interface {
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)                                                                          //perm:read
 
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)                                             //perm:read
-	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)                                 //perm:read
+	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)                                 //perm:read
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) //perm:read
 
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error) //perm:read

api/api_gateway.go

@@ -114,7 +114,7 @@ type Gateway interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)

api/mocks/mock_full.go

@@ -1042,7 +1042,7 @@ func (mr *MockFullNodeMockRecorder) EthChainId(arg0 interface{}) *gomock.Call {
 }
 
 // EthEstimateGas mocks base method.
-func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+func (m *MockFullNode) EthEstimateGas(arg0 context.Context, arg1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "EthEstimateGas", arg0, arg1)
 	ret0, _ := ret[0].(ethtypes.EthUint64)

api/proxy_gen.go

@@ -255,7 +255,7 @@ type FullNodeMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) `perm:"read"`
 
-	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) `perm:"read"`
+	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) `perm:"read"`
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) `perm:"read"`
 
@@ -679,7 +679,7 @@ type GatewayMethods struct {
 
 	EthChainId func(p0 context.Context) (ethtypes.EthUint64, error) ``
 
-	EthEstimateGas func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) ``
+	EthEstimateGas func(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) ``
 
 	EthFeeHistory func(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthFeeHistory, error) ``
 
@@ -2134,14 +2134,14 @@ func (s *FullNodeStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+func (s *FullNodeStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+func (s *FullNodeStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
@@ -4400,14 +4400,14 @@ func (s *GatewayStub) EthChainId(p0 context.Context) (ethtypes.EthUint64, error)
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 
-func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+func (s *GatewayStruct) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	if s.Internal.EthEstimateGas == nil {
 		return *new(ethtypes.EthUint64), ErrNotSupported
 	}
 	return s.Internal.EthEstimateGas(p0, p1)
 }
 
-func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 jsonrpc.RawParams) (ethtypes.EthUint64, error) {
+func (s *GatewayStub) EthEstimateGas(p0 context.Context, p1 ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	return *new(ethtypes.EthUint64), ErrNotSupported
 }
 

api/types.go

@@ -56,17 +56,9 @@ type PubsubScore struct {
 	Score *pubsub.PeerScoreSnapshot
 }
 
-// MessageSendSpec contains optional fields which modify message sending behavior
 type MessageSendSpec struct {
-	// MaxFee specifies a cap on network fees related to this message
-	MaxFee abi.TokenAmount
-
-	// MsgUuid specifies a unique message identifier which can be used on node (or node cluster)
-	// level to prevent double-sends of messages even when nonce generation is not handled by sender
+	MaxFee  abi.TokenAmount
 	MsgUuid uuid.UUID
-
-	// MaximizeFeeCap makes message FeeCap be based entirely on MaxFee
-	MaximizeFeeCap bool
 }
 
 type MpoolMessageWhole struct {

blockstore/idstore.go

@@ -183,17 +183,3 @@ func (b *idstore) Close() error {
 func (b *idstore) Flush(ctx context.Context) error {
 	return b.bs.Flush(ctx)
 }
-
-func (b *idstore) CollectGarbage(ctx context.Context, options ...BlockstoreGCOption) error {
-	if bs, ok := b.bs.(BlockstoreGC); ok {
-		return bs.CollectGarbage(ctx, options...)
-	}
-	return xerrors.Errorf("not supported")
-}
-
-func (b *idstore) GCOnce(ctx context.Context, options ...BlockstoreGCOption) error {
-	if bs, ok := b.bs.(BlockstoreGCOnce); ok {
-		return bs.GCOnce(ctx, options...)
-	}
-	return xerrors.Errorf("not supported")
-}

blockstore/splitstore/splitstore.go

@@ -182,6 +182,7 @@ type SplitStore struct {
 
 	compactionIndex int64
 	pruneIndex      int64
+	onlineGCCnt     int64
 
 	ctx    context.Context
 	cancel func()

blockstore/splitstore/splitstore_compact.go

@@ -66,9 +66,8 @@ var (
 )
 
 const (
-	batchSize              = 16384
-	cidKeySize             = 128
-	purgeWorkSliceDuration = time.Second
+	batchSize  = 16384
+	cidKeySize = 128
 )
 
 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -1373,21 +1372,9 @@ func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet
 		return err
 	}
 
-	now := time.Now()
-
 	err := coldr.ForEach(func(c cid.Cid) error {
 		batch = append(batch, c)
 		if len(batch) == batchSize {
-			// add some time slicing to the purge as this a very disk I/O heavy operation that
-			// requires write access to txnLk that may starve other operations that require
-			// access to the blockstore.
-			elapsed := time.Since(now)
-			if elapsed > purgeWorkSliceDuration {
-				// work 1 slice, sleep 4 slices, or 20% utilization
-				time.Sleep(4 * elapsed)
-				now = time.Now()
-			}
-
 			return deleteBatch()
 		}
 

build/bootstrap/mainnet.pi

@@ -1,6 +1,9 @@
 /dns4/bootstrap-0.mainnet.filops.net/tcp/1347/p2p/12D3KooWCVe8MmsEMes2FzgTpt9fXtmCY7wrq91GRiaC8PHSCCBj
 /dns4/bootstrap-1.mainnet.filops.net/tcp/1347/p2p/12D3KooWCwevHg1yLCvktf2nvLu7L9894mcrJR4MsBCcm4syShVc
 /dns4/bootstrap-2.mainnet.filops.net/tcp/1347/p2p/12D3KooWEWVwHGn2yR36gKLozmb4YjDJGerotAPGxmdWZx2nxMC4
+/dns4/bootstrap-3.mainnet.filops.net/tcp/1347/p2p/12D3KooWKhgq8c7NQ9iGjbyK7v7phXvG6492HQfiDaGHLHLQjk7R
+/dns4/bootstrap-4.mainnet.filops.net/tcp/1347/p2p/12D3KooWL6PsFNPhYftrJzGgF5U18hFoaVhfGk7xwzD8yVrHJ3Uc
+/dns4/bootstrap-5.mainnet.filops.net/tcp/1347/p2p/12D3KooWLFynvDQiUpXoHroV1YxKHhPJgysQGH2k3ZGwtWzR4dFH
 /dns4/bootstrap-6.mainnet.filops.net/tcp/1347/p2p/12D3KooWP5MwCiqdMETF9ub1P3MbCvQCcfconnYHbWg6sUJcDRQQ
 /dns4/bootstrap-7.mainnet.filops.net/tcp/1347/p2p/12D3KooWRs3aY1p3juFjPy8gPN95PEQChm2QKGUCAdcDCC4EBMKf
 /dns4/bootstrap-8.mainnet.filops.net/tcp/1347/p2p/12D3KooWScFR7385LTyR4zU1bYdzSiiAb5rnNABfVahPvVSzyTkR
@@ -8,5 +11,7 @@
 /dns4/bootstrap-0.starpool.in/tcp/12757/p2p/12D3KooWGHpBMeZbestVEWkfdnC9u7p6uFHXL1n7m1ZBqsEmiUzz
 /dns4/bootstrap-1.starpool.in/tcp/12757/p2p/12D3KooWQZrGH1PxSNZPum99M1zNvjNFM33d1AAu5DcvdHptuU7u
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
+/dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
+/dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
 /dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
 /dns4/bootstrap-venus.mainnet.filincubator.com/tcp/8888/p2p/QmQu8C6deXwKvJP2D8B6QGyhngc3ZiDnFzEHBDx8yeBXST

build/isnearupgrade.go

@@ -5,8 +5,5 @@ import (
 )
 
 func IsNearUpgrade(epoch, upgradeEpoch abi.ChainEpoch) bool {
-	if upgradeEpoch < 0 {
-		return false
-	}
 	return epoch > upgradeEpoch-Finality && epoch < upgradeEpoch+Finality
 }

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.25.1-dev"
+const BuildVersion = "1.25.1"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/actors/builtin/miner/state.go.template

@@ -72,7 +72,7 @@ func (s *state{{.v}}) AvailableBalance(bal abi.TokenAmount) (available abi.Token
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available{{if (ge .v 2)}}, err{{end}} = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v0.go

@@ -62,7 +62,7 @@ func (s *state0) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v10.go

@@ -62,7 +62,7 @@ func (s *state10) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v11.go

@@ -62,7 +62,7 @@ func (s *state11) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v12.go

@@ -62,7 +62,7 @@ func (s *state12) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmou
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v2.go

@@ -61,7 +61,7 @@ func (s *state2) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v3.go

@@ -62,7 +62,7 @@ func (s *state3) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v4.go

@@ -62,7 +62,7 @@ func (s *state4) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v5.go

@@ -62,7 +62,7 @@ func (s *state5) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v6.go

@@ -62,7 +62,7 @@ func (s *state6) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v7.go

@@ -62,7 +62,7 @@ func (s *state7) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v8.go

@@ -62,7 +62,7 @@ func (s *state8) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/miner/v9.go

@@ -62,7 +62,7 @@ func (s *state9) AvailableBalance(bal abi.TokenAmount) (available abi.TokenAmoun
 			available = abi.NewTokenAmount(0)
 		}
 	}()
-	// this panics if the miner doesn't have enough funds to cover their locked pledge
+	// this panics if the miner doesnt have enough funds to cover their locked pledge
 	available, err = s.GetAvailableBalance(bal)
 	return available, err
 }

chain/actors/builtin/paych/actor.go.template

@@ -29,7 +29,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-// Load returns an abstract copy of payment channel state, regardless of actor version
+// Load returns an abstract copy of payment channel state, irregardless of actor version
 func Load(store adt.Store, act *types.Actor) (State, error) {
 	if name, av, ok := actors.GetActorMetaByCode(act.Code); ok {
        if name != manifest.PaychKey {

chain/actors/builtin/paych/paych.go

@@ -29,7 +29,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-// Load returns an abstract copy of payment channel state, regardless of actor version
+// Load returns an abstract copy of payment channel state, irregardless of actor version
 func Load(store adt.Store, act *types.Actor) (State, error) {
 	if name, av, ok := actors.GetActorMetaByCode(act.Code); ok {
 		if name != manifest.PaychKey {

chain/actors/params.go

@@ -13,7 +13,7 @@ import (
 func SerializeParams(i cbg.CBORMarshaler) ([]byte, aerrors.ActorError) {
 	buf := new(bytes.Buffer)
 	if err := i.MarshalCBOR(buf); err != nil {
-		// TODO: shouldn't this be a fatal error?
+		// TODO: shouldnt this be a fatal error?
 		return nil, aerrors.Absorb(err, exitcode.ErrSerialization, "failed to encode parameter")
 	}
 	return buf.Bytes(), nil

chain/beacon/drand/drand.go

@@ -29,6 +29,19 @@ import (
 
 var log = logging.Logger("drand")
 
+type drandPeer struct {
+	addr string
+	tls  bool
+}
+
+func (dp *drandPeer) Address() string {
+	return dp.addr
+}
+
+func (dp *drandPeer) IsTLS() bool {
+	return dp.tls
+}
+
 // DrandBeacon connects Lotus with a drand network in order to provide
 // randomness to the system in a way that's aligned with Filecoin rounds/epochs.
 //

chain/beacon/drand/drand_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func TestPrintGroupInfo(t *testing.T) {
-	server := build.DrandConfigs[build.DrandTestnet].Servers[0]
+	server := build.DrandConfigs[build.DrandDevnet].Servers[0]
 	c, err := hclient.New(server, nil, nil)
 	assert.NoError(t, err)
 	cg := c.(interface {
@@ -31,7 +31,7 @@ func TestPrintGroupInfo(t *testing.T) {
 
 func TestMaxBeaconRoundForEpoch(t *testing.T) {
 	todayTs := uint64(1652222222)
-	db, err := NewDrandBeacon(todayTs, build.BlockDelaySecs, nil, build.DrandConfigs[build.DrandTestnet])
+	db, err := NewDrandBeacon(todayTs, build.BlockDelaySecs, nil, build.DrandConfigs[build.DrandDevnet])
 	assert.NoError(t, err)
 	mbr15 := db.MaxBeaconRoundForEpoch(network.Version15, 100)
 	mbr16 := db.MaxBeaconRoundForEpoch(network.Version16, 100)

chain/consensus/compute_state.go

@@ -147,6 +147,9 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
 				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
+		if ret.ExitCode != 0 {
+			return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
+		}
 
 		return nil
 	}

chain/consensus/filcns/filecoin.go

@@ -91,6 +91,9 @@ var RewardFunc = func(ctx context.Context, vmi vm.Interface, em stmgr.ExecMonito
 		}
 	}
 
+	if ret.ExitCode != 0 {
+		return xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
+	}
 	return nil
 }
 

chain/events/events_height.go

@@ -180,7 +180,7 @@ func (e *heightEventsObserver) Revert(ctx context.Context, from, to *types.TipSe
 	// Update the head first so we don't accidental skip reverting a concurrent call to ChainAt.
 	e.updateHead(to)
 
-	// Call revert on all heights between the two tipsets, handling empty tipsets.
+	// Call revert on all hights between the two tipsets, handling empty tipsets.
 	for h := from.Height(); h > to.Height(); h-- {
 		e.lk.Lock()
 		triggers := e.tsHeights[h]

chain/events/events_test.go

@@ -673,7 +673,7 @@ func TestCalled(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing gets randomly called
+	// create few blocks to make sure nothing get's randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -991,7 +991,7 @@ func TestCalledNull(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing gets randomly called
+	// create few blocks to make sure nothing get's randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -1050,7 +1050,7 @@ func TestRemoveTriggersOnMessage(t *testing.T) {
 	}, 3, 20, matchAddrMethod(t0123, 5))
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing gets randomly called
+	// create few blocks to make sure nothing get's randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)
@@ -1155,7 +1155,7 @@ func TestStateChanged(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	// create few blocks to make sure nothing gets randomly called
+	// create few blocks to make sure nothing get's randomly called
 
 	fcs.advance(0, 4, 0, nil) // H=5
 	require.Equal(t, false, applied)

chain/events/filter/index.go

@@ -326,7 +326,7 @@ func NewEventIndex(ctx context.Context, path string, chainStore *store.ChainStor
 		}
 
 		if version == 2 {
-			log.Infof("upgrading event index from version 2 to version 3")
+			log.Infof("upgrading event index from version 1 to version 2")
 
 			// to upgrade to version 3 we only need to create an index on the event_entry.key column
 			// and on the event.emitter_addr column

chain/events/state/ctxstore.go

@@ -0,0 +1,25 @@
+package state
+
+import (
+	"context"
+
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+)
+
+type contextStore struct {
+	ctx context.Context
+	cst *cbor.BasicIpldStore
+}
+
+func (cs *contextStore) Context() context.Context {
+	return cs.ctx
+}
+
+func (cs *contextStore) Get(ctx context.Context, c cid.Cid, out interface{}) error {
+	return cs.cst.Get(ctx, c, out)
+}
+
+func (cs *contextStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) {
+	return cs.cst.Put(ctx, v)
+}

chain/exchange/cbor_gen.go

@@ -306,9 +306,9 @@ func (t *Response) UnmarshalCBOR(r io.Reader) (err error) {
 	return nil
 }
 
-var lengthBufCompactedMessages = []byte{132}
+var lengthBufCompactedMessagesCBOR = []byte{132}
 
-func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+func (t *CompactedMessagesCBOR) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
 		return err
@@ -316,12 +316,12 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write(lengthBufCompactedMessages); err != nil {
+	if _, err := cw.Write(lengthBufCompactedMessagesCBOR); err != nil {
 		return err
 	}
 
 	// t.Bls ([]*types.Message) (slice)
-	if len(t.Bls) > cbg.MaxLength {
+	if len(t.Bls) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Bls was too long")
 	}
 
@@ -334,7 +334,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 	if len(t.BlsIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
 	}
@@ -343,24 +343,13 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.BlsIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 
 	// t.Secpk ([]*types.SignedMessage) (slice)
-	if len(t.Secpk) > cbg.MaxLength {
+	if len(t.Secpk) > 150000 {
 		return xerrors.Errorf("Slice value in field t.Secpk was too long")
 	}
 
@@ -373,7 +362,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 	if len(t.SecpkIncludes) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
 	}
@@ -382,26 +371,15 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 	for _, v := range t.SecpkIncludes {
-		if len(v) > cbg.MaxLength {
-			return xerrors.Errorf("Slice value in field v was too long")
-		}
-
-		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
-		for _, v := range v {
-
-			if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
-				return err
-			}
-
-		}
 	}
 	return nil
 }
 
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
-	*t = CompactedMessages{}
+func (t *CompactedMessagesCBOR) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = CompactedMessagesCBOR{}
 
 	cr := cbg.NewCborReader(r)
 
@@ -430,7 +408,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Bls: array too large (%d)", extra)
 	}
 
@@ -471,7 +449,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.BlsIncludes ([][]uint64) (slice)
+	// t.BlsIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -487,7 +465,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.BlsIncludes = make([][]uint64, extra)
+		t.BlsIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -499,47 +477,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.BlsIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.BlsIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.BlsIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.BlsIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.BlsIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 
@@ -550,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		return err
 	}
 
-	if extra > cbg.MaxLength {
+	if extra > 150000 {
 		return fmt.Errorf("t.Secpk: array too large (%d)", extra)
 	}
 
@@ -591,7 +535,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 		}
 	}
 
-	// t.SecpkIncludes ([][]uint64) (slice)
+	// t.SecpkIncludes ([]exchange.messageIndices) (slice)
 
 	maj, extra, err = cr.ReadHeader()
 	if err != nil {
@@ -607,7 +551,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	}
 
 	if extra > 0 {
-		t.SecpkIncludes = make([][]uint64, extra)
+		t.SecpkIncludes = make([]messageIndices, extra)
 	}
 
 	for i := 0; i < int(extra); i++ {
@@ -619,47 +563,13 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 			_ = extra
 			_ = err
 
-			maj, extra, err = cr.ReadHeader()
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.MaxLength {
-				return fmt.Errorf("t.SecpkIncludes[i]: array too large (%d)", extra)
-			}
-
-			if maj != cbg.MajArray {
-				return fmt.Errorf("expected cbor array")
-			}
-
-			if extra > 0 {
-				t.SecpkIncludes[i] = make([]uint64, extra)
-			}
-
-			for j := 0; j < int(extra); j++ {
-				{
-					var maj byte
-					var extra uint64
-					var err error
-					_ = maj
-					_ = extra
-					_ = err
-
-					{
-
-						maj, extra, err = cr.ReadHeader()
-						if err != nil {
-							return err
-						}
-						if maj != cbg.MajUnsignedInt {
-							return fmt.Errorf("wrong type for uint64 field")
-						}
-						t.SecpkIncludes[i][j] = uint64(extra)
-
-					}
+				if err := t.SecpkIncludes[i].UnmarshalCBOR(cr); err != nil {
+					return xerrors.Errorf("unmarshaling t.SecpkIncludes[i]: %w", err)
 				}
-			}
 
+			}
 		}
 	}
 

chain/exchange/client.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"time"
 
@@ -23,6 +24,10 @@ import (
 	"github.com/filecoin-project/lotus/lib/peermgr"
 )
 
+// Set the max exchange message size to 120MiB. Purely based on gas numbers, we can include ~8MiB of
+// messages per block, so I've set this to 120MiB to be _very_ safe.
+const maxExchangeMessageSize = (15 * 8) << 20
+
 // client implements exchange.Client, using the libp2p ChainExchange protocol
 // as the fetching mechanism.
 type client struct {
@@ -242,7 +247,7 @@ func (c *client) processResponse(req *Request, res *Response, tipsets []*types.T
 			// If we didn't request the headers they should have been provided
 			// by the caller.
 			if len(tipsets) < len(res.Chain) {
-				return nil, xerrors.Errorf("not enough tipsets provided for message response validation, needed %d, have %d", len(res.Chain), len(tipsets))
+				return nil, xerrors.Errorf("not enought tipsets provided for message response validation, needed %d, have %d", len(res.Chain), len(tipsets))
 			}
 			chain := make([]*BSTipSet, 0, resLength)
 			for i, resChain := range res.Chain {
@@ -381,7 +386,7 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 	defer span.End()
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
-			trace.StringAttribute("peer", peer.String()),
+			trace.StringAttribute("peer", peer.Pretty()),
 		)
 	}
 	defer func() {
@@ -434,10 +439,11 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 		log.Warnw("CloseWrite err", "error", err)
 	}
 
-	// Read response.
+	// Read response, limiting the size of the response to maxExchangeMessageSize as we allow a
+	// lot of messages (10k+) but they'll mostly be quite small.
 	var res Response
 	err = cborutil.ReadCborRPC(
-		bufio.NewReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline)),
+		bufio.NewReader(io.LimitReader(incrt.New(stream, ReadResMinSpeed, ReadResDeadline), maxExchangeMessageSize)),
 		&res)
 	if err != nil {
 		c.peerTracker.logFailure(peer, build.Clock.Since(connectionStart), req.Length)

chain/exchange/protocol.go

@@ -154,6 +154,8 @@ type BSTipSet struct {
 // FIXME: The logic to decompress this structure should belong
 //
 //	to itself, not to the consumer.
+//
+// NOTE: Max messages is: BlockMessageLimit (10k) * MaxTipsetSize (15) = 150k
 type CompactedMessages struct {
 	Bls         []*types.Message
 	BlsIncludes [][]uint64

chain/exchange/protocol_encoding.go

@@ -0,0 +1,125 @@
+package exchange
+
+import (
+	"fmt"
+	"io"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	xerrors "golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/build"
+	types "github.com/filecoin-project/lotus/chain/types"
+)
+
+// Type used for encoding/decoding compacted messages. This is a ustom type as we need custom limits.
+// - Max messages is 150,000 as that's 15 times the max block size (in messages). It needs to be
+// large enough to cover a full tipset full of full blocks.
+type CompactedMessagesCBOR struct {
+	Bls         []*types.Message `cborgen:"maxlen=150000"`
+	BlsIncludes []messageIndices
+
+	Secpk         []*types.SignedMessage `cborgen:"maxlen=150000"`
+	SecpkIncludes []messageIndices
+}
+
+// Unmarshal into the "decoding" struct, then copy into the actual struct.
+func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
+	var c CompactedMessagesCBOR
+	if err := c.UnmarshalCBOR(r); err != nil {
+		return err
+	}
+	t.Bls = c.Bls
+	t.BlsIncludes = make([][]uint64, len(c.BlsIncludes))
+	for i, v := range c.BlsIncludes {
+		t.BlsIncludes[i] = v.v
+	}
+	t.Secpk = c.Secpk
+	t.SecpkIncludes = make([][]uint64, len(c.SecpkIncludes))
+	for i, v := range c.SecpkIncludes {
+		t.SecpkIncludes[i] = v.v
+	}
+	return nil
+}
+
+// Copy into the encoding struct, then marshal.
+func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	var c CompactedMessagesCBOR
+	c.Bls = t.Bls
+	c.BlsIncludes = make([]messageIndices, len(t.BlsIncludes))
+	for i, v := range t.BlsIncludes {
+		c.BlsIncludes[i].v = v
+	}
+	c.Secpk = t.Secpk
+	c.SecpkIncludes = make([]messageIndices, len(t.SecpkIncludes))
+	for i, v := range t.SecpkIncludes {
+		c.SecpkIncludes[i].v = v
+	}
+	return c.MarshalCBOR(w)
+}
+
+// this needs to be a struct or cborgen will peak into it and ignore the Unmarshal/Marshal functions
+type messageIndices struct {
+	v []uint64
+}
+
+func (t *messageIndices) UnmarshalCBOR(r io.Reader) (err error) {
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra > uint64(build.BlockMessageLimit) {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	if extra > 0 {
+		t.v = make([]uint64, extra)
+	}
+
+	for i := 0; i < int(extra); i++ {
+		maj, extra, err := cr.ReadHeader()
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.v[i] = extra
+
+	}
+	return nil
+}
+
+func (t *messageIndices) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if len(t.v) > build.BlockMessageLimit {
+		return xerrors.Errorf("Slice value in field v was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.v))); err != nil {
+		return err
+	}
+	for _, v := range t.v {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, v); err != nil {
+			return err
+		}
+	}
+	return nil
+}

chain/index/msgindex.go

@@ -131,7 +131,7 @@ func NewMsgIndex(lctx context.Context, basePath string, cs ChainStore) (MsgIndex
 
 	db, err := sql.Open("sqlite3", dbPath)
 	if err != nil {
-		// TODO [nice to have]: automatically delete corrupt databases
+		// TODO [nice to have]: automaticaly delete corrupt databases
 		//      but for now we can just error and let the operator delete.
 		return nil, xerrors.Errorf("error opening msgindex database: %w", err)
 	}

chain/market/store.go

@@ -39,6 +39,23 @@ func (ps *Store) save(ctx context.Context, state *FundedAddressState) error {
 	return ps.ds.Put(ctx, k, b)
 }
 
+// get the state for the given address
+func (ps *Store) get(ctx context.Context, addr address.Address) (*FundedAddressState, error) {
+	k := dskeyForAddr(addr)
+
+	data, err := ps.ds.Get(ctx, k)
+	if err != nil {
+		return nil, err
+	}
+
+	var state FundedAddressState
+	err = cborrpc.ReadCborRPC(bytes.NewReader(data), &state)
+	if err != nil {
+		return nil, err
+	}
+	return &state, nil
+}
+
 // forEach calls iter with each address in the datastore
 func (ps *Store) forEach(ctx context.Context, iter func(*FundedAddressState)) error {
 	res, err := ps.ds.Query(ctx, dsq.Query{Prefix: dsKeyAddr})

chain/messagepool/messagepool.go

@@ -21,7 +21,6 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/minio/blake2b-simd"
 	"github.com/raulk/clock"
-	"go.opencensus.io/stats"
 	"golang.org/x/xerrors"
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
@@ -211,10 +210,8 @@ func ComputeRBF(curPrem abi.TokenAmount, replaceByFeeRatio types.Percent) abi.To
 
 func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.MessageSendSpec) {
 	var maxFee abi.TokenAmount
-	var maximizeFeeCap bool
 	if sendSpec != nil {
 		maxFee = sendSpec.MaxFee
-		maximizeFeeCap = sendSpec.MaximizeFeeCap
 	}
 	if maxFee.Int == nil || maxFee.Equals(big.Zero()) {
 		mf, err := mff()
@@ -225,12 +222,15 @@ func CapGasFee(mff dtypes.DefaultMaxFeeFunc, msg *types.Message, sendSpec *api.M
 		maxFee = mf
 	}
 
-	gaslimit := types.NewInt(uint64(msg.GasLimit))
-	totalFee := types.BigMul(msg.GasFeeCap, gaslimit)
-	if maximizeFeeCap || totalFee.GreaterThan(maxFee) {
-		msg.GasFeeCap = big.Div(maxFee, gaslimit)
+	gl := types.NewInt(uint64(msg.GasLimit))
+	totalFee := types.BigMul(msg.GasFeeCap, gl)
+
+	if totalFee.LessThanEqual(maxFee) {
+		msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
+		return
 	}
 
+	msg.GasFeeCap = big.Div(maxFee, gl)
 	msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
 }
 
@@ -1022,9 +1022,6 @@ func (mp *MessagePool) addLocked(ctx context.Context, m *types.SignedMessage, st
 		}
 	})
 
-	// Record the current size of the Mpool
-	stats.Record(ctx, metrics.MpoolMessageCount.M(int64(mp.currentSize)))
-
 	return nil
 }
 
@@ -1217,9 +1214,6 @@ func (mp *MessagePool) remove(ctx context.Context, from address.Address, nonce u
 			return
 		}
 	}
-
-	// Record the current size of the Mpool
-	stats.Record(ctx, metrics.MpoolMessageCount.M(int64(mp.currentSize)))
 }
 
 func (mp *MessagePool) Pending(ctx context.Context) ([]*types.SignedMessage, *types.TipSet) {

chain/state/statetree.go

@@ -570,7 +570,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error
 			}
 
 			// no need to record anything here, there are no duplicates in the actors HAMT
-			// itself.
+			// iself.
 			if _, ok := seen[addr]; ok {
 				return nil
 			}
@@ -588,7 +588,7 @@ func (st *StateTree) ForEach(f func(address.Address, *types.Actor) error) error
 		}
 
 		// no need to record anything here, there are no duplicates in the actors HAMT
-		// itself.
+		// iself.
 		if _, ok := seen[addr]; ok {
 			return nil
 		}

chain/stmgr/forks.go

@@ -235,6 +235,11 @@ func (sm *StateManager) hasExpensiveForkBetween(parent, height abi.ChainEpoch) b
 	return false
 }
 
+func (sm *StateManager) hasExpensiveFork(height abi.ChainEpoch) bool {
+	_, ok := sm.expensiveUpgrades[height]
+	return ok
+}
+
 func runPreMigration(ctx context.Context, sm *StateManager, fn PreMigrationFunc, cache *nv16.MemMigrationCache, ts *types.TipSet) {
 	height := ts.Height()
 	parent := ts.ParentState()

chain/store/index_test.go

@@ -42,7 +42,7 @@ func TestIndexSeeks(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, syncds.MutexWrap(datastore.NewMapDatastore()), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	_, _, err = cs.Import(ctx, bytes.NewReader(gencar))
+	_, err = cs.Import(ctx, bytes.NewReader(gencar))
 	if err != nil {
 		t.Fatal(err)
 	}

chain/store/messages.go

@@ -212,8 +212,13 @@ func (cs *ChainStore) MessagesForTipset(ctx context.Context, ts *types.TipSet) (
 
 	var out []types.ChainMsg
 	for _, bm := range bmsgs {
-		out = append(out, bm.BlsMessages...)
-		out = append(out, bm.SecpkMessages...)
+		for _, blsm := range bm.BlsMessages {
+			out = append(out, blsm)
+		}
+
+		for _, secm := range bm.SecpkMessages {
+			out = append(out, secm)
+		}
 	}
 
 	return out, nil

chain/store/snapshot.go

@@ -60,7 +60,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo
 	})
 }
 
-func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipSet, genesis *types.BlockHeader, err error) {
+func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (*types.TipSet, error) {
 	// TODO: writing only to the state blockstore is incorrect.
 	//  At this time, both the state and chain blockstores are backed by the
 	//  universal store. When we physically segregate the stores, we will need
@@ -69,7 +69,7 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipS
 
 	br, err := carv2.NewBlockReader(r)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("loadcar failed: %w", err)
+		return nil, xerrors.Errorf("loadcar failed: %w", err)
 	}
 
 	s := cs.StateBlockstore()
@@ -80,51 +80,27 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipS
 		putThrottle <- nil
 	}
 
-	if len(br.Roots) == 0 {
-		return nil, nil, xerrors.Errorf("no roots in snapshot car file")
-	}
-	nextTailCid := br.Roots[0]
-
-	var tailBlock types.BlockHeader
-	tailBlock.Height = abi.ChainEpoch(-1)
-
 	var buf []blocks.Block
 	for {
 		blk, err := br.Next()
 		if err != nil {
-
-			// we're at the end
 			if err == io.EOF {
 				if len(buf) > 0 {
 					if err := s.PutMany(ctx, buf); err != nil {
-						return nil, nil, err
+						return nil, err
 					}
 				}
 
 				break
 			}
-			return nil, nil, err
+			return nil, err
 		}
 
-		// check for header block, looking for genesis
-		if blk.Cid() == nextTailCid && tailBlock.Height != 0 {
-			if err := tailBlock.UnmarshalCBOR(bytes.NewReader(blk.RawData())); err != nil {
-				return nil, nil, xerrors.Errorf("failed to unmarshal genesis block: %w", err)
-			}
-			if len(tailBlock.Parents) > 0 {
-				nextTailCid = tailBlock.Parents[0]
-			} else {
-				// note: even the 0th block has a parent linking to the cbor genesis block
-				return nil, nil, xerrors.Errorf("current block (epoch %d cid %s) has no parents", tailBlock.Height, tailBlock.Cid())
-			}
-		}
-
-		// append to batch
 		buf = append(buf, blk)
 
 		if len(buf) > 1000 {
 			if lastErr := <-putThrottle; lastErr != nil { // consume one error to have the right to add one
-				return nil, nil, lastErr
+				return nil, lastErr
 			}
 
 			go func(buf []blocks.Block) {
@@ -137,17 +113,13 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipS
 	// check errors
 	for i := 0; i < parallelPuts; i++ {
 		if lastErr := <-putThrottle; lastErr != nil {
-			return nil, nil, lastErr
+			return nil, lastErr
 		}
 	}
 
-	if tailBlock.Height != 0 {
-		return nil, nil, xerrors.Errorf("expected genesis block to have height 0 (genesis), got %d: %s", tailBlock.Height, tailBlock.Cid())
-	}
-
 	root, err := cs.LoadTipSet(ctx, types.NewTipSetKey(br.Roots...))
 	if err != nil {
-		return nil, nil, xerrors.Errorf("failed to load root tipset from chainfile: %w", err)
+		return nil, xerrors.Errorf("failed to load root tipset from chainfile: %w", err)
 	}
 
 	ts := root
@@ -163,10 +135,10 @@ func (cs *ChainStore) Import(ctx context.Context, r io.Reader) (head *types.TipS
 	}
 
 	if err := cs.PersistTipsets(ctx, tssToPersist); err != nil {
-		return nil, nil, xerrors.Errorf("failed to persist tipsets: %w", err)
+		return nil, xerrors.Errorf("failed to persist tipsets: %w", err)
 	}
 
-	return root, &tailBlock, nil
+	return root, nil
 }
 
 type walkSchedTaskType int
@@ -195,7 +167,7 @@ func (t walkSchedTaskType) String() string {
 	case dagTask:
 		return "dag"
 	}
-	panic(fmt.Sprintf("unknown task %d", t))
+	panic(fmt.Sprintf("unknow task %d", t))
 }
 
 type walkTask struct {
@@ -684,7 +656,9 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 		}
 
 		if b.Height > 0 {
-			blocksToWalk = append(blocksToWalk, b.Parents...)
+			for _, p := range b.Parents {
+				blocksToWalk = append(blocksToWalk, p)
+			}
 		} else {
 			// include the genesis block
 			cids = append(cids, b.Parents...)

chain/store/store_test.go

@@ -118,7 +118,7 @@ func TestChainExportImport(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, _, err := cs.Import(context.TODO(), buf)
+	root, err := cs.Import(context.TODO(), buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +153,7 @@ func TestChainImportTipsetKeyCid(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, datastore.NewMapDatastore(), filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, _, err := cs.Import(ctx, buf)
+	root, err := cs.Import(ctx, buf)
 	require.NoError(t, err)
 
 	require.Truef(t, root.Equals(last), "imported chain differed from exported chain")
@@ -202,7 +202,7 @@ func TestChainExportImportFull(t *testing.T) {
 	cs := store.NewChainStore(nbs, nbs, ds, filcns.Weight, nil)
 	defer cs.Close() //nolint:errcheck
 
-	root, _, err := cs.Import(context.TODO(), buf)
+	root, err := cs.Import(context.TODO(), buf)
 	if err != nil {
 		t.Fatal(err)
 	}

chain/sub/incoming.go

@@ -516,7 +516,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 		return pubsub.ValidationReject
 	}
 	if len(idxrMsg.ExtraData) == 0 {
-		log.Debugw("ignoring message missing miner id", "peer", originPeer)
+		log.Debugw("ignoring messsage missing miner id", "peer", originPeer)
 		return pubsub.ValidationIgnore
 	}
 
@@ -552,7 +552,7 @@ func (v *IndexerMessageValidator) Validate(ctx context.Context, pid peer.ID, msg
 		// Check that the miner ID maps to the peer that sent the message.
 		err = v.authenticateMessage(ctx, minerAddr, originPeer)
 		if err != nil {
-			log.Warnw("cannot authenticate message", "err", err, "peer", originPeer, "minerID", minerAddr)
+			log.Warnw("cannot authenticate messsage", "err", err, "peer", originPeer, "minerID", minerAddr)
 			stats.Record(ctx, metrics.IndexerMessageValidationFailure.M(1))
 			return pubsub.ValidationReject
 		}

chain/sub/ratelimit/queue_test.go

@@ -1,61 +0,0 @@
-package ratelimit
-
-import (
-	"testing"
-)
-
-func TestQueue(t *testing.T) {
-	const size = 3
-	q := &queue{buf: make([]int64, size)}
-
-	if q.len() != 0 {
-		t.Fatalf("q.len() = %d, expect 0", q.len())
-	}
-
-	if q.cap() != size {
-		t.Fatalf("q.cap() = %d, expect %d", q.cap(), size)
-	}
-
-	for i := int64(0); i < int64(size); i++ {
-		err := q.push(i)
-		if err != nil {
-			t.Fatalf("cannot push element %d", i)
-		}
-	}
-
-	if q.len() != size {
-		t.Fatalf("q.len() = %d, expect %d", q.len(), size)
-	}
-
-	err := q.push(int64(size))
-	if err != ErrRateLimitExceeded {
-		t.Fatalf("pushing element beyond capacity should have failed with err: %s, got %s", ErrRateLimitExceeded, err)
-	}
-
-	if q.front() != 0 {
-		t.Fatalf("q.front() = %d, expect 0", q.front())
-	}
-
-	if q.back() != int64(size-1) {
-		t.Fatalf("q.back() = %d, expect %d", q.back(), size-1)
-	}
-
-	popVal := q.pop()
-	if popVal != 0 {
-		t.Fatalf("q.pop() = %d, expect 0", popVal)
-	}
-
-	if q.len() != size-1 {
-		t.Fatalf("q.len() = %d, expect %d", q.len(), size-1)
-	}
-
-	// Testing truncation.
-	threshold := int64(1)
-	q.truncate(threshold)
-	if q.len() != 1 {
-		t.Fatalf("q.len() after truncate = %d, expect 1", q.len())
-	}
-	if q.front() != 2 {
-		t.Fatalf("q.front() after truncate = %d, expect 2", q.front())
-	}
-}

chain/sync.go

@@ -844,7 +844,7 @@ loop:
 		return nil, xerrors.Errorf("failed to load next local tipset: %w", err)
 	}
 	if base.IsChildOf(knownParent) {
-		// common case: receiving a block that's potentially part of the same tipset as our best block
+		// common case: receiving a block thats potentially part of the same tipset as our best block
 		return blockSet, nil
 	}
 

chain/sync_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/libp2p/go-libp2p/core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
 
@@ -343,6 +344,13 @@ func (tu *syncTestUtil) addClientNode() int {
 	return len(tu.nds) - 1
 }
 
+func (tu *syncTestUtil) pid(n int) peer.ID {
+	nal, err := tu.nds[n].NetAddrsListen(tu.ctx)
+	require.NoError(tu.t, err)
+
+	return nal.ID
+}
+
 func (tu *syncTestUtil) connect(from, to int) {
 	toPI, err := tu.nds[to].NetAddrsListen(tu.ctx)
 	require.NoError(tu.t, err)

chain/types/blockmsg_test.go

@@ -7,6 +7,9 @@ import (
 )
 
 func TestDecodeBlockMsg(t *testing.T) {
+	type args struct {
+		b []byte
+	}
 	tests := []struct {
 		name    string
 		data    []byte

chain/types/ethtypes/eth_transactions.go

@@ -62,14 +62,9 @@ type EthTxArgs struct {
 // - BlockHash
 // - BlockNumber
 // - TransactionIndex
+// - From
 // - Hash
 func EthTxFromSignedEthMessage(smsg *types.SignedMessage) (EthTx, error) {
-	// The from address is always an f410f address, never an ID or other address.
-	if !IsEthAddress(smsg.Message.From) {
-		return EthTx{}, xerrors.Errorf("sender must be an eth account, was %s", smsg.Message.From)
-	}
-
-	// Probably redundant, but we might as well check.
 	if smsg.Signature.Type != typescrypto.SigTypeDelegated {
 		return EthTx{}, xerrors.Errorf("signature is not delegated type, is type: %d", smsg.Signature.Type)
 	}
@@ -84,18 +79,10 @@ func EthTxFromSignedEthMessage(smsg *types.SignedMessage) (EthTx, error) {
 		return EthTx{}, xerrors.Errorf("failed to recover signature: %w", err)
 	}
 
-	from, err := EthAddressFromFilecoinAddress(smsg.Message.From)
-	if err != nil {
-		// This should be impossible as we've already asserted that we have an EthAddress
-		// sender...
-		return EthTx{}, xerrors.Errorf("sender was not an eth account")
-	}
-
 	return EthTx{
 		Nonce:                EthUint64(txArgs.Nonce),
 		ChainID:              EthUint64(txArgs.ChainID),
 		To:                   txArgs.To,
-		From:                 from,
 		Value:                EthBigInt(txArgs.Value),
 		Type:                 Eip1559TxType,
 		Gas:                  EthUint64(txArgs.GasLimit),

chain/types/ethtypes/eth_types.go

@@ -799,45 +799,6 @@ func GetContractEthAddressFromCode(sender EthAddress, salt [32]byte, initcode []
 	return ethAddr, nil
 }
 
-// EthEstimateGasParams handles raw jsonrpc params for eth_estimateGas
-type EthEstimateGasParams struct {
-	Tx       EthCall
-	BlkParam *EthBlockNumberOrHash
-}
-
-func (e *EthEstimateGasParams) UnmarshalJSON(b []byte) error {
-	var params []json.RawMessage
-	err := json.Unmarshal(b, &params)
-	if err != nil {
-		return err
-	}
-
-	switch len(params) {
-	case 2:
-		err = json.Unmarshal(params[1], &e.BlkParam)
-		if err != nil {
-			return err
-		}
-		fallthrough
-	case 1:
-		err = json.Unmarshal(params[0], &e.Tx)
-		if err != nil {
-			return err
-		}
-	default:
-		return xerrors.Errorf("expected 1 or 2 params, got %d", len(params))
-	}
-
-	return nil
-}
-
-func (e EthEstimateGasParams) MarshalJSON() ([]byte, error) {
-	if e.BlkParam != nil {
-		return json.Marshal([]interface{}{e.Tx, e.BlkParam})
-	}
-	return json.Marshal([]interface{}{e.Tx})
-}
-
 // EthFeeHistoryParams handles raw jsonrpc params for eth_feeHistory
 type EthFeeHistoryParams struct {
 	BlkCount          EthUint64

chain/types/message_receipt_cbor.go

@@ -140,7 +140,7 @@ func (t *messageReceiptV0) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative overflow")
+				return fmt.Errorf("int64 negative oveflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -186,7 +186,7 @@ func (t *messageReceiptV0) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative overflow")
+				return fmt.Errorf("int64 negative oveflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -278,7 +278,7 @@ func (t *messageReceiptV1) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative overflow")
+				return fmt.Errorf("int64 negative oveflow")
 			}
 			extraI = -1 - extraI
 		default:
@@ -324,7 +324,7 @@ func (t *messageReceiptV1) UnmarshalCBOR(r io.Reader) (err error) {
 		case cbg.MajNegativeInt:
 			extraI = int64(extra)
 			if extraI < 0 {
-				return fmt.Errorf("int64 negative overflow")
+				return fmt.Errorf("int64 negative oveflow")
 			}
 			extraI = -1 - extraI
 		default:

chain/types/vmcontext.go

@@ -27,3 +27,24 @@ type StateTree interface {
 
 	Version() StateTreeVersion
 }
+
+type storageWrapper struct {
+	s Storage
+}
+
+func (sw *storageWrapper) Put(i cbg.CBORMarshaler) (cid.Cid, error) {
+	c, err := sw.s.Put(i)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	return c, nil
+}
+
+func (sw *storageWrapper) Get(c cid.Cid, out cbg.CBORUnmarshaler) error {
+	if err := sw.s.Get(c, out); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/net.go

@@ -282,7 +282,7 @@ var NetDisconnect = &cli.Command{
 				fmt.Println("failure")
 				return err
 			}
-			fmt.Printf("disconnect %s: ", pid)
+			fmt.Printf("disconnect %s: ", pid.Pretty())
 			err = api.NetDisconnect(ctx, pid)
 			if err != nil {
 				fmt.Println("failure")
@@ -312,7 +312,7 @@ var NetConnect = &cli.Command{
 		}
 
 		for _, pi := range pis {
-			fmt.Printf("connect %s: ", pi.ID)
+			fmt.Printf("connect %s: ", pi.ID.Pretty())
 			err := api.NetConnect(ctx, pi)
 			if err != nil {
 				fmt.Println("failure")

cli/state.go

@@ -1920,29 +1920,8 @@ var StateSysActorCIDsCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-
-		var actorsCidTuples []struct {
-			actorName string
-			actorCid  cid.Cid
-		}
-
-		for name, actorCid := range actorsCids {
-			keyVal := struct {
-				actorName string
-				actorCid  cid.Cid
-			}{
-				actorName: name,
-				actorCid:  actorCid,
-			}
-			actorsCidTuples = append(actorsCidTuples, keyVal)
-		}
-
-		sort.Slice(actorsCidTuples, func(i, j int) bool {
-			return actorsCidTuples[i].actorName < actorsCidTuples[j].actorName
-		})
-
-		for _, keyVal := range actorsCidTuples {
-			_, _ = fmt.Fprintf(tw, "%v\t%v\n", keyVal.actorName, keyVal.actorCid)
+		for name, cid := range actorsCids {
+			_, _ = fmt.Fprintf(tw, "%v\t%v\n", name, cid)
 		}
 		return tw.Flush()
 	},

cli/sync.go

@@ -14,7 +14,6 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
-	cliutil "github.com/filecoin-project/lotus/cli/util"
 )
 
 var SyncCmd = &cli.Command{
@@ -263,9 +262,6 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 	}
 	firstApp = state.VMApplied
 
-	// eta computes the ETA for the sync to complete (with a lookback of 10 processed items)
-	eta := cliutil.NewETA(10)
-
 	for {
 		state, err := napi.SyncState(ctx)
 		if err != nil {
@@ -316,10 +312,8 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 			fmt.Print("\r\x1b[2K\x1b[A")
 		}
 
-		todo := theight - ss.Height
-
 		fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", workerID, baseHeight, theight, heightDiff)
-		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d, ETA: %s\n", ss.Stage, ss.Height, todo, eta.Update(int64(todo)))
+		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d\n", ss.Stage, ss.Height, theight-ss.Height)
 		lastLines = 2
 
 		if i%samples == 0 {

cli/util/eta.go

@@ -1,94 +0,0 @@
-package cliutil
-
-import (
-	"fmt"
-	"math"
-	"time"
-)
-
-// ETA implements a very simple eta calculator based on the number of remaining items. It does not
-// require knowing the work size in advance and is therefore suitable for streaming workloads and
-// also does not require that consecutive updates have a monotonically decreasing remaining value.
-type ETA struct {
-	// max number of items to keep in memory
-	maxItems int
-	// a queue of most recently updated items
-	items []item
-	// we store the last calculated ETA which we reuse if there was not change in remaining items
-	lastETA string
-}
-
-type item struct {
-	timestamp time.Time
-	remaining int64
-}
-
-// NewETA creates a new ETA calculator of the given size
-func NewETA(maxItems int) *ETA {
-	return &ETA{
-		maxItems: maxItems,
-		items:    make([]item, 0),
-	}
-}
-
-// Update updates the ETA calculator with the remaining number of items and returns the ETA
-func (e *ETA) Update(remaining int64) string {
-	item := item{
-		timestamp: time.Now(),
-		remaining: remaining,
-	}
-
-	if len(e.items) == 0 {
-		e.items = append(e.items, item)
-		return ""
-	}
-
-	if e.items[len(e.items)-1].remaining == remaining {
-		// we ignore updates with the same remaining value and just return the previous ETA
-		return e.lastETA
-	} else if e.items[len(e.items)-1].remaining < remaining {
-		// remaining went up from previous update, lets estimate how many items were processed using the
-		// average number processed items in the queue.
-		var avgProcessedPerItem int64 = 1
-		if len(e.items) > 1 {
-			diffRemaining := e.items[0].remaining - e.items[len(e.items)-1].remaining
-			avgProcessedPerItem = int64(math.Round(float64(diffRemaining) / float64(len(e.items))))
-		}
-
-		// diff is the difference in increase in remaining since last update plus the average number of processed
-		// items we estimate that were processed this round
-		diff := remaining - e.items[len(e.items)-1].remaining + avgProcessedPerItem
-
-		// we update all items in the queue by shifting their remaining value accordingly. This means that we
-		// always have strictly decreasing remaining values in the queue
-		for i := range e.items {
-			e.items[i].remaining += diff
-		}
-	}
-
-	// append the item to the queue and remove the oldest item if needed
-	if len(e.items) >= e.maxItems {
-		e.items = e.items[1:]
-	}
-	e.items = append(e.items, item)
-
-	// calculate the average processing time per item in the queue
-	diffMs := e.items[len(e.items)-1].timestamp.Sub(e.items[0].timestamp).Milliseconds()
-	nrItemsProcessed := e.items[0].remaining - e.items[len(e.items)-1].remaining
-	avg := diffMs / nrItemsProcessed
-
-	// use that average processing time to estimate how long the remaining items will take
-	// and cache that ETA so we don't have to recalculate it on every update unless the
-	// remaining value changes
-	e.lastETA = msToETA(avg * remaining)
-
-	return e.lastETA
-}
-
-func msToETA(ms int64) string {
-	seconds := ms / 1000
-	minutes := seconds / 60
-	hours := minutes / 60
-
-	return fmt.Sprintf("%02dh:%02dm:%02ds", hours, minutes%60, seconds%60)
-}

cli/wallet.go

@@ -22,7 +22,6 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/tablewriter"
 )
@@ -460,12 +459,7 @@ var walletSign = &cli.Command{
 		sig, err := api.WalletSign(ctx, addr, msg)
 
 		if err != nil {
-			// Check if the address is a multisig address
-			act, actErr := api.StateGetActor(ctx, addr, types.EmptyTSK)
-			if actErr == nil && builtin.IsMultisigActor(act.Code) {
-				return xerrors.Errorf("specified signer address is a multisig actor, it doesn’t have keys to sign transactions. To send a message with a multisig, signers of the multisig need to propose and approve transactions.")
-			}
-			return xerrors.Errorf("failed to sign message: %w", err)
+			return err
 		}
 
 		sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)

cmd/lotus-bench/cli.go

@@ -1,312 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/urfave/cli/v2"
-)
-
-var cliCmd = &cli.Command{
-	Name:  "cli",
-	Usage: "Runs a concurrent stress test on one or more binaries commands and prints the performance metrics including latency distribution and histogram",
-	Description: `This benchmark has the following features:
-* Can query each command both sequentially and concurrently
-* Supports rate limiting
-* Can query multiple different commands at once (supporting different concurrency level and rate limiting for each command)
-* Gives a nice reporting summary of the stress testing of each command (including latency distribution, histogram and more)
-* Easy to use
-
-To use this benchmark you must specify the commands you want to test using the --cmd options, the format of it is:
-
-  --cmd=CMD[:CONCURRENCY][:QPS] where only NAME is required.
-
-Here are some real examples:
-  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' // runs the command with default concurrency and qps
-  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3'  // override concurrency to 3
-  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages::100' // override to 100 qps while using default concurrency
-  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3:100' // run using 3 workers but limit to 100 qps
-  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' --cmd='lotus sync wait' // run two commands at once
-`,
-	Flags: []cli.Flag{
-		&cli.DurationFlag{
-			Name:  "duration",
-			Value: 60 * time.Second,
-			Usage: "Duration of benchmark in seconds",
-		},
-		&cli.IntFlag{
-			Name:  "concurrency",
-			Value: 10,
-			Usage: "How many workers should be used per command (can be overridden per command)",
-		},
-		&cli.IntFlag{
-			Name:  "qps",
-			Value: 0,
-			Usage: "How many requests per second should be sent per command (can be overridden per command), a value of 0 means no limit",
-		},
-		&cli.StringSliceFlag{
-			Name:  "cmd",
-			Usage: `Command to benchmark, you can specify multiple commands by repeating this flag. You can also specify command specific options to set the concurrency and qps for each command (see usage).`,
-		},
-		&cli.DurationFlag{
-			Name:  "watch",
-			Value: 0 * time.Second,
-			Usage: "If >0 then generates reports every N seconds (only supports linux/unix)",
-		},
-		&cli.BoolFlag{
-			Name:  "print-response",
-			Value: false,
-			Usage: "print the response of each request",
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		if len(cctx.StringSlice("cmd")) == 0 {
-			return errors.New("you must specify and least one cmd to benchmark")
-		}
-
-		var cmds []*CMD
-		for _, str := range cctx.StringSlice("cmd") {
-			entries := strings.SplitN(str, ":", 3)
-			if len(entries) == 0 {
-				return errors.New("invalid cmd format")
-			}
-
-			// check if concurrency was specified
-			concurrency := cctx.Int("concurrency")
-			if len(entries) > 1 {
-				if len(entries[1]) > 0 {
-					var err error
-					concurrency, err = strconv.Atoi(entries[1])
-					if err != nil {
-						return fmt.Errorf("could not parse concurrency value from command %s: %v", entries[0], err)
-					}
-				}
-			}
-
-			// check if qps was specified
-			qps := cctx.Int("qps")
-			if len(entries) > 2 {
-				if len(entries[2]) > 0 {
-					var err error
-					qps, err = strconv.Atoi(entries[2])
-					if err != nil {
-						return fmt.Errorf("could not parse qps value from command %s: %v", entries[0], err)
-					}
-				}
-			}
-
-			cmds = append(cmds, &CMD{
-				w:           os.Stdout,
-				cmd:         entries[0],
-				concurrency: concurrency,
-				qps:         qps,
-				printResp:   cctx.Bool("print-response"),
-			})
-		}
-
-		// terminate early on ctrl+c
-		c := make(chan os.Signal, 1)
-		signal.Notify(c, os.Interrupt)
-		go func() {
-			<-c
-			fmt.Println("Received interrupt, stopping...")
-			for _, cmd := range cmds {
-				cmd.Stop()
-			}
-		}()
-
-		// stop all threads after duration
-		go func() {
-			time.Sleep(cctx.Duration("duration"))
-			for _, cmd := range cmds {
-				cmd.Stop()
-			}
-		}()
-
-		// start all threads
-		var wg sync.WaitGroup
-		wg.Add(len(cmds))
-
-		for _, cmd := range cmds {
-			go func(cmd *CMD) {
-				defer wg.Done()
-				err := cmd.Run()
-				if err != nil {
-					fmt.Printf("error running cmd: %v\n", err)
-				}
-			}(cmd)
-		}
-
-		// if watch is set then print a report every N seconds
-		var progressCh chan struct{}
-		if cctx.Duration("watch") > 0 {
-			progressCh = make(chan struct{}, 1)
-			go func(progressCh chan struct{}) {
-				ticker := time.NewTicker(cctx.Duration("watch"))
-				for {
-					clearAndPrintReport := func() {
-						// clear the screen move the cursor to the top left
-						fmt.Print("\033[2J")
-						fmt.Printf("\033[%d;%dH", 1, 1)
-						for i, cmd := range cmds {
-							cmd.Report()
-							if i < len(cmds)-1 {
-								fmt.Println()
-							}
-						}
-					}
-					select {
-					case <-ticker.C:
-						clearAndPrintReport()
-					case <-progressCh:
-						clearAndPrintReport()
-						return
-					}
-				}
-			}(progressCh)
-		}
-
-		wg.Wait()
-
-		if progressCh != nil {
-			// wait for the watch go routine to return
-			progressCh <- struct{}{}
-
-			// no need to print the report again
-			return nil
-		}
-
-		// print the report for each command
-		for i, cmd := range cmds {
-			cmd.Report()
-			if i < len(cmds)-1 {
-				fmt.Println()
-			}
-		}
-
-		return nil
-	},
-}
-
-// CMD handles the benchmarking of a single command.
-type CMD struct {
-	w io.Writer
-	// the cmd we want to benchmark
-	cmd string
-	// the number of concurrent requests to make to this command
-	concurrency int
-	// if >0 then limit to qps is the max number of requests per second to make to this command (0 = no limit)
-	qps int
-	// whether or not to print the response of each request (useful for debugging)
-	printResp bool
-	// instruct the worker go routines to stop
-	stopCh chan struct{}
-	// when the command bencharking started
-	start time.Time
-	// results channel is used by the workers to send results to the reporter
-	results chan *result
-	// reporter handles reading the results from workers and printing the report statistics
-	reporter *Reporter
-}
-
-func (c *CMD) Run() error {
-	var wg sync.WaitGroup
-	wg.Add(c.concurrency)
-
-	c.results = make(chan *result, c.concurrency*1_000)
-	c.stopCh = make(chan struct{}, c.concurrency)
-
-	go func() {
-		c.reporter = NewReporter(c.results, c.w)
-		c.reporter.Run()
-	}()
-
-	c.start = time.Now()
-
-	// throttle the number of requests per second
-	var qpsTicker *time.Ticker
-	if c.qps > 0 {
-		qpsTicker = time.NewTicker(time.Second / time.Duration(c.qps))
-	}
-
-	for i := 0; i < c.concurrency; i++ {
-		go func() {
-			c.startWorker(qpsTicker)
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-
-	// close the results channel so reporter will stop
-	close(c.results)
-
-	// wait until the reporter is done
-	<-c.reporter.doneCh
-
-	return nil
-}
-
-func (c *CMD) startWorker(qpsTicker *time.Ticker) {
-	for {
-		// check if we should stop
-		select {
-		case <-c.stopCh:
-			return
-		default:
-		}
-
-		// wait for the next tick if we are rate limiting this command
-		if qpsTicker != nil {
-			<-qpsTicker.C
-		}
-
-		start := time.Now()
-
-		var statusCode int = 0
-
-		arr := strings.Fields(c.cmd)
-
-		data, err := exec.Command(arr[0], arr[1:]...).Output()
-		if err != nil {
-			fmt.Println("1")
-			if exitError, ok := err.(*exec.ExitError); ok {
-				statusCode = exitError.ExitCode()
-			} else {
-				statusCode = 1
-			}
-		} else {
-			if c.printResp {
-				fmt.Printf("[%s] %s", c.cmd, string(data))
-			}
-		}
-
-		c.results <- &result{
-			statusCode: &statusCode,
-			err:        err,
-			duration:   time.Since(start),
-		}
-	}
-}
-
-func (c *CMD) Stop() {
-	for i := 0; i < c.concurrency; i++ {
-		c.stopCh <- struct{}{}
-	}
-}
-
-func (c *CMD) Report() {
-	total := time.Since(c.start)
-	fmt.Fprintf(c.w, "[%s]:\n", c.cmd)
-	fmt.Fprintf(c.w, "- Options:\n")
-	fmt.Fprintf(c.w, "  - concurrency: %d\n", c.concurrency)
-	fmt.Fprintf(c.w, "  - qps: %d\n", c.qps)
-	c.reporter.Print(total, c.w)
-}

cmd/lotus-bench/import.go

@@ -304,7 +304,7 @@ var importBenchCmd = &cli.Command{
 				return fmt.Errorf("no CAR file provided for import")
 			}
 
-			head, _, err = cs.Import(cctx.Context, carFile)
+			head, err = cs.Import(cctx.Context, carFile)
 			if err != nil {
 				return err
 			}
@@ -497,6 +497,21 @@ type Invocation struct {
 
 const GasPerNs = 10
 
+func countGasCosts(et *types.ExecutionTrace) int64 {
+	var cgas int64
+
+	for _, gc := range et.GasCharges {
+		cgas += gc.ComputeGas
+	}
+
+	for _, sub := range et.Subcalls {
+		c := countGasCosts(&sub) //nolint
+		cgas += c
+	}
+
+	return cgas
+}
+
 type stats struct {
 	timeTaken meanVar
 	gasRatio  meanVar

cmd/lotus-bench/main.go

@@ -120,7 +120,6 @@ func main() {
 			sealBenchCmd,
 			simpleCmd,
 			importBenchCmd,
-			cliCmd,
 			rpcCmd,
 		},
 	}

cmd/lotus-bench/reporter.go

@@ -1,181 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-	"text/tabwriter"
-	"time"
-)
-
-// result is the result of a single rpc method request.
-type result struct {
-	err        error
-	statusCode *int
-	duration   time.Duration
-}
-
-// Reporter reads the results from the workers through the results channel and aggregates the results.
-type Reporter struct {
-	// write the report to this writer
-	w io.Writer
-	// the reporter read the results from this channel
-	results chan *result
-	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
-	doneCh chan bool
-
-	// lock protect the following fields during critical sections (if --watch was specified)
-	lock sync.Mutex
-	// the latencies of all requests
-	latencies []int64
-	// the number of requests that returned each status code
-	statusCodes map[int]int
-	// the number of errors that occurred
-	errors map[string]int
-}
-
-func NewReporter(results chan *result, w io.Writer) *Reporter {
-	return &Reporter{
-		w:           w,
-		results:     results,
-		doneCh:      make(chan bool, 1),
-		statusCodes: make(map[int]int),
-		errors:      make(map[string]int),
-	}
-}
-
-func (r *Reporter) Run() {
-	for res := range r.results {
-		r.lock.Lock()
-
-		r.latencies = append(r.latencies, res.duration.Milliseconds())
-
-		if res.statusCode != nil {
-			r.statusCodes[*res.statusCode]++
-		}
-
-		if res.err != nil {
-			if len(r.errors) < 1_000_000 {
-				r.errors[res.err.Error()]++
-			} else {
-				// we don't want to store too many errors in memory
-				r.errors["hidden"]++
-			}
-		} else {
-			r.errors["nil"]++
-		}
-
-		r.lock.Unlock()
-	}
-
-	r.doneCh <- true
-}
-
-func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
-	r.lock.Lock()
-	defer r.lock.Unlock()
-
-	nrReq := int64(len(r.latencies))
-	if nrReq == 0 {
-		fmt.Println("No requests were made")
-		return
-	}
-
-	// we need to sort the latencies slice to calculate the percentiles
-	sort.Slice(r.latencies, func(i, j int) bool {
-		return r.latencies[i] < r.latencies[j]
-	})
-
-	var totalLatency int64 = 0
-	for _, latency := range r.latencies {
-		totalLatency += latency
-	}
-
-	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
-	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
-	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
-	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
-	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
-	fmt.Fprintf(w, "- Latency distribution:\n")
-	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
-	for _, p := range percentiles {
-		idx := int64(p * float64(nrReq))
-		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
-	}
-
-	// create a simple histogram with 10 buckets spanning the range of latency
-	// into equal ranges
-	//
-	nrBucket := 10
-	buckets := make([]Bucket, nrBucket)
-	latencyRange := r.latencies[len(r.latencies)-1]
-	bucketRange := latencyRange / int64(nrBucket)
-
-	// mark the end of each bucket
-	for i := 0; i < nrBucket; i++ {
-		buckets[i].start = int64(i) * bucketRange
-		buckets[i].end = buckets[i].start + bucketRange
-		// extend the last bucked by any remaning range caused by the integer division
-		if i == nrBucket-1 {
-			buckets[i].end = latencyRange
-		}
-	}
-
-	// count the number of requests in each bucket
-	currBucket := 0
-	for i := 0; i < len(r.latencies); {
-		if r.latencies[i] <= buckets[currBucket].end {
-			buckets[currBucket].cnt++
-			i++
-		} else {
-			currBucket++
-		}
-	}
-
-	// print the histogram using a tabwriter which will align the columns nicely
-	fmt.Fprintf(w, "- Histogram:\n")
-	const padding = 2
-	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
-	for i := 0; i < nrBucket; i++ {
-		ratio := float64(buckets[i].cnt) / float64(nrReq)
-		bars := strings.Repeat("#", int(ratio*100))
-		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
-	}
-	tabWriter.Flush() //nolint:errcheck
-
-	fmt.Fprintf(w, "- Status codes:\n")
-	for code, cnt := range r.statusCodes {
-		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
-	}
-
-	// print the 10 most occurring errors (in case error values are not unique)
-	//
-	type kv struct {
-		err string
-		cnt int
-	}
-	var sortedErrors []kv
-	for err, cnt := range r.errors {
-		sortedErrors = append(sortedErrors, kv{err, cnt})
-	}
-	sort.Slice(sortedErrors, func(i, j int) bool {
-		return sortedErrors[i].cnt > sortedErrors[j].cnt
-	})
-	fmt.Fprintf(w, "- Errors (top 10):\n")
-	for i, se := range sortedErrors {
-		if i > 10 {
-			break
-		}
-		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
-	}
-}
-
-type Bucket struct {
-	start int64
-	// the end value of the bucket
-	end int64
-	// how many entries are in the bucket
-	cnt int
-}

cmd/lotus-bench/rpc.go

@@ -9,9 +9,11 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
+	"text/tabwriter"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -241,6 +243,13 @@ type RPCMethod struct {
 	reporter *Reporter
 }
 
+// result is the result of a single rpc method request.
+type result struct {
+	err        error
+	statusCode *int
+	duration   time.Duration
+}
+
 func (rpc *RPCMethod) Run() error {
 	client := &http.Client{
 		Timeout: 0,
@@ -402,3 +411,166 @@ func (rpc *RPCMethod) Report() {
 	fmt.Fprintf(rpc.w, "  - qps: %d\n", rpc.qps)
 	rpc.reporter.Print(total, rpc.w)
 }
+
+// Reporter reads the results from the workers through the results channel and aggregates the results.
+type Reporter struct {
+	// write the report to this writer
+	w io.Writer
+	// the reporter read the results from this channel
+	results chan *result
+	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
+	doneCh chan bool
+
+	// lock protect the following fields during critical sections (if --watch was specified)
+	lock sync.Mutex
+	// the latencies of all requests
+	latencies []int64
+	// the number of requests that returned each status code
+	statusCodes map[int]int
+	// the number of errors that occurred
+	errors map[string]int
+}
+
+func NewReporter(results chan *result, w io.Writer) *Reporter {
+	return &Reporter{
+		w:           w,
+		results:     results,
+		doneCh:      make(chan bool, 1),
+		statusCodes: make(map[int]int),
+		errors:      make(map[string]int),
+	}
+}
+
+func (r *Reporter) Run() {
+	for res := range r.results {
+		r.lock.Lock()
+
+		r.latencies = append(r.latencies, res.duration.Milliseconds())
+
+		if res.statusCode != nil {
+			r.statusCodes[*res.statusCode]++
+		}
+
+		if res.err != nil {
+			if len(r.errors) < 1_000_000 {
+				r.errors[res.err.Error()]++
+			} else {
+				// we don't want to store too many errors in memory
+				r.errors["hidden"]++
+			}
+		} else {
+			r.errors["nil"]++
+		}
+
+		r.lock.Unlock()
+	}
+
+	r.doneCh <- true
+}
+
+func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	nrReq := int64(len(r.latencies))
+	if nrReq == 0 {
+		fmt.Println("No requests were made")
+		return
+	}
+
+	// we need to sort the latencies slice to calculate the percentiles
+	sort.Slice(r.latencies, func(i, j int) bool {
+		return r.latencies[i] < r.latencies[j]
+	})
+
+	var totalLatency int64 = 0
+	for _, latency := range r.latencies {
+		totalLatency += latency
+	}
+
+	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
+	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
+	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
+	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
+	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
+	fmt.Fprintf(w, "- Latency distribution:\n")
+	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
+	for _, p := range percentiles {
+		idx := int64(p * float64(nrReq))
+		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
+	}
+
+	// create a simple histogram with 10 buckets spanning the range of latency
+	// into equal ranges
+	//
+	nrBucket := 10
+	buckets := make([]Bucket, nrBucket)
+	latencyRange := r.latencies[len(r.latencies)-1]
+	bucketRange := latencyRange / int64(nrBucket)
+
+	// mark the end of each bucket
+	for i := 0; i < nrBucket; i++ {
+		buckets[i].start = int64(i) * bucketRange
+		buckets[i].end = buckets[i].start + bucketRange
+		// extend the last bucked by any remaning range caused by the integer division
+		if i == nrBucket-1 {
+			buckets[i].end = latencyRange
+		}
+	}
+
+	// count the number of requests in each bucket
+	currBucket := 0
+	for i := 0; i < len(r.latencies); {
+		if r.latencies[i] <= buckets[currBucket].end {
+			buckets[currBucket].cnt++
+			i++
+		} else {
+			currBucket++
+		}
+	}
+
+	// print the histogram using a tabwriter which will align the columns nicely
+	fmt.Fprintf(w, "- Histogram:\n")
+	const padding = 2
+	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
+	for i := 0; i < nrBucket; i++ {
+		ratio := float64(buckets[i].cnt) / float64(nrReq)
+		bars := strings.Repeat("#", int(ratio*100))
+		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
+	}
+	tabWriter.Flush() //nolint:errcheck
+
+	fmt.Fprintf(w, "- Status codes:\n")
+	for code, cnt := range r.statusCodes {
+		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
+	}
+
+	// print the 10 most occurring errors (in case error values are not unique)
+	//
+	type kv struct {
+		err string
+		cnt int
+	}
+	var sortedErrors []kv
+	for err, cnt := range r.errors {
+		sortedErrors = append(sortedErrors, kv{err, cnt})
+	}
+	sort.Slice(sortedErrors, func(i, j int) bool {
+		return sortedErrors[i].cnt > sortedErrors[j].cnt
+	})
+	fmt.Fprintf(w, "- Errors (top 10):\n")
+	for i, se := range sortedErrors {
+		if i > 10 {
+			break
+		}
+		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
+	}
+}
+
+type Bucket struct {
+	start int64
+	// the end value of the bucket
+	end int64
+	// how many entries are in the bucket
+	cnt int
+}

cmd/lotus-bench/simple.go

@@ -266,10 +266,7 @@ var simplePreCommit1 = &cli.Command{
 			ProofType: spt(sectorSize, cctx.Bool("synthetic")),
 		}
 
-		ticket := [32]byte{}
-		for i := range ticket {
-			ticket[i] = 1
-		}
+		var ticket [32]byte // all zero
 
 		pieces, err := ParsePieceInfos(cctx, 3)
 		if err != nil {
@@ -308,36 +305,7 @@ var simplePreCommit2 = &cli.Command{
 			Name:  "synthetic",
 			Usage: "generate synthetic PoRep proofs",
 		},
-		&cli.StringFlag{
-			Name:  "external-pc2",
-			Usage: "command for computing PC2 externally",
-		},
 	},
-	Description: `Compute PreCommit2 inputs and seal a sector.
-
---external-pc2 can be used to compute the PreCommit2 inputs externally.
-The flag behaves similarly to the related lotus-worker flag, using it in
-lotus-bench may be useful for testing if the external PreCommit2 command is
-invoked correctly.
-
-The command will be called with a number of environment variables set:
-* EXTSEAL_PC2_SECTOR_NUM: the sector number
-* EXTSEAL_PC2_SECTOR_MINER: the miner id
-* EXTSEAL_PC2_PROOF_TYPE: the proof type
-* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
-* EXTSEAL_PC2_CACHE: the path to the cache directory
-* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
-* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
-
-The command is expected to:
-* Create cache sc-02-data-tree-r* files
-* Create cache sc-02-data-tree-c* files
-* Create cache p_aux / t_aux files
-* Transform the sealed file in place
-
-Example invocation of lotus-bench as external executor:
-'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
-`,
 	ArgsUsage: "[sealed] [cache] [pc1 out]",
 	Action: func(cctx *cli.Context) error {
 		ctx := cctx.Context
@@ -362,18 +330,7 @@ Example invocation of lotus-bench as external executor:
 			storiface.FTSealed: cctx.Args().Get(0),
 			storiface.FTCache:  cctx.Args().Get(1),
 		}
-
-		var opts []ffiwrapper.FFIWrapperOpt
-
-		if cctx.IsSet("external-pc2") {
-			extSeal := ffiwrapper.ExternalSealer{
-				PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")),
-			}
-
-			opts = append(opts, ffiwrapper.WithExternalSealCalls(extSeal))
-		}
-
-		sealer, err := ffiwrapper.New(pp, opts...)
+		sealer, err := ffiwrapper.New(pp)
 		if err != nil {
 			return err
 		}
@@ -463,12 +420,7 @@ var simpleCommit1 = &cli.Command{
 
 		start := time.Now()
 
-		ticket := [32]byte{}
-		seed := [32]byte{}
-		for i := range ticket {
-			ticket[i] = 1
-			seed[i] = 1
-		}
+		var ticket, seed [32]byte // all zero
 
 		commd, err := cid.Parse(cctx.Args().Get(2))
 		if err != nil {
@@ -698,10 +650,6 @@ var simpleWinningPost = &cli.Command{
 			Usage: "pass miner address (only necessary if using existing sectorbuilder)",
 			Value: "t01000",
 		},
-		&cli.BoolFlag{
-			Name:  "show-inputs",
-			Usage: "output inputs for winning post generation",
-		},
 	},
 	ArgsUsage: "[sealed] [cache] [comm R] [sector num]",
 	Action: func(cctx *cli.Context) error {
@@ -772,17 +720,6 @@ var simpleWinningPost = &cli.Command{
 		fmt.Printf("Vanilla %s (%s)\n", challenge.Sub(start), bps(sectorSize, 1, challenge.Sub(start)))
 		fmt.Printf("Proof %s (%s)\n", end.Sub(challenge), bps(sectorSize, 1, end.Sub(challenge)))
 		fmt.Println(base64.StdEncoding.EncodeToString(proof[0].ProofBytes))
-
-		if cctx.Bool("show-inputs") {
-			fmt.Println("GenerateWinningPoStWithVanilla info:")
-
-			fmt.Printf(" wpt: %d\n", wpt)
-			fmt.Printf(" mid: %d\n", mid)
-			fmt.Printf(" rand: %x\n", rand)
-			fmt.Printf(" vp: %x\n", vp)
-			fmt.Printf(" proof: %x\n", proof)
-		}
-
 		return nil
 	},
 }

cmd/lotus-gateway/main.go

@@ -143,7 +143,7 @@ var runCmd = &cli.Command{
 		},
 		&cli.DurationFlag{
 			Name:  "rate-limit-timeout",
-			Usage: "the maximum time to wait for the rate limiter before returning an error to clients",
+			Usage: "the maximum time to wait for the rate limter before returning an error to clients",
 			Value: gateway.DefaultRateLimitTimeout,
 		},
 		&cli.Int64Flag{

cmd/lotus-miner/sectors.go

@@ -2566,8 +2566,6 @@ var sectorsUnsealCmd = &cli.Command{
 			return xerrors.Errorf("could not parse sector number: %w", err)
 		}
 
-		fmt.Printf("Unsealing sector %d\n", sectorNum)
-
 		return minerAPI.SectorUnseal(ctx, abi.SectorNumber(sectorNum))
 	},
 }

cmd/lotus-shed/genesis-verify.go

@@ -62,7 +62,7 @@ var genesisVerifyCmd = &cli.Command{
 			return xerrors.Errorf("opening the car file: %w", err)
 		}
 
-		ts, _, err := cs.Import(cctx.Context, f)
+		ts, err := cs.Import(cctx.Context, f)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/mpool.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/urfave/cli/v2"
 
@@ -44,20 +43,10 @@ var minerSelectMsgsCmd = &cli.Command{
 			return err
 		}
 
-		// Get the size of the mempool
-		pendingMsgs, err := api.MpoolPending(ctx, types.EmptyTSK)
-		if err != nil {
-			return err
-		}
-		mpoolSize := len(pendingMsgs)
-
-		// Measure the time taken by MpoolSelect
-		startTime := time.Now()
 		msgs, err := api.MpoolSelect(ctx, head.Key(), cctx.Float64("ticket-quality"))
 		if err != nil {
 			return err
 		}
-		duration := time.Since(startTime)
 
 		var totalGas int64
 		for i, f := range msgs {
@@ -75,9 +64,6 @@ var minerSelectMsgsCmd = &cli.Command{
 			totalGas += f.Message.GasLimit
 		}
 
-		// Log the duration, size of the mempool, selected messages and total gas limit of selected messages
-		fmt.Printf("Message selection took %s\n", duration)
-		fmt.Printf("Size of the mempool: %d\n", mpoolSize)
 		fmt.Println("selected messages: ", len(msgs))
 		fmt.Printf("total gas limit of selected messages: %d / %d (%0.2f%%)\n", totalGas, build.BlockGasLimit, 100*float64(totalGas)/float64(build.BlockGasLimit))
 		return nil

cmd/lotus-shed/pruning.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 
-	"github.com/filecoin-project/lotus/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/node/repo"
@@ -144,6 +144,13 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}()
 
+		// After migrating to native blockstores, this has been made
+		// database-specific.
+		badgbs, ok := bs.(*badgerbs.Blockstore)
+		if !ok {
+			return fmt.Errorf("only badger blockstores are supported")
+		}
+
 		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
@@ -153,12 +160,8 @@ var stateTreePruneCmd = &cli.Command{
 		const DiscardRatio = 0.2
 		if cctx.Bool("only-ds-gc") {
 			fmt.Println("running datastore gc....")
-			gbs, ok := bs.(blockstore.BlockstoreGCOnce)
-			if !ok {
-				return xerrors.Errorf("blockstore %T does not support GC", bs)
-			}
 			for i := 0; i < cctx.Int("gc-count"); i++ {
-				if err := gbs.GCOnce(ctx, blockstore.WithThreshold(DiscardRatio)); err != nil {
+				if err := badgbs.DB().RunValueLogGC(DiscardRatio); err != nil {
 					return xerrors.Errorf("datastore GC failed: %w", err)
 				}
 			}
@@ -205,6 +208,13 @@ var stateTreePruneCmd = &cli.Command{
 			return nil
 		}
 
+		b := badgbs.DB().NewWriteBatch()
+		defer b.Cancel()
+
+		markForRemoval := func(c cid.Cid) error {
+			return b.Delete(badgbs.StorageKey(nil, c))
+		}
+
 		keys, err := bs.AllKeysChan(context.Background())
 		if err != nil {
 			return xerrors.Errorf("failed to query blockstore: %w", err)
@@ -215,12 +225,12 @@ var stateTreePruneCmd = &cli.Command{
 		var deleteCount int
 		var goodHits int
 		for k := range keys {
-			if goodSet.Has(k) {
+			if goodSet.HasRaw(k.Bytes()) {
 				goodHits++
 				continue
 			}
 
-			if err := bs.DeleteBlock(ctx, k); err != nil {
+			if err := markForRemoval(k); err != nil {
 				return fmt.Errorf("failed to remove cid %s: %w", k, err)
 			}
 
@@ -233,15 +243,13 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}
 
-		fmt.Println("running datastore gc....")
-		gbs, ok := bs.(blockstore.BlockstoreGCOnce)
-		if !ok {
-			fmt.Println("gc not supported...")
-			return nil
+		if err := b.Flush(); err != nil {
+			return xerrors.Errorf("failed to flush final batch delete: %w", err)
 		}
 
+		fmt.Println("running datastore gc....")
 		for i := 0; i < cctx.Int("gc-count"); i++ {
-			if err := gbs.GCOnce(ctx, blockstore.WithThreshold(DiscardRatio)); err != nil {
+			if err := badgbs.DB().RunValueLogGC(DiscardRatio); err != nil {
 				return xerrors.Errorf("datastore GC failed: %w", err)
 			}
 		}

cmd/lotus-worker/main.go

@@ -7,7 +7,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -39,7 +38,6 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/storage/paths"
 	"github.com/filecoin-project/lotus/storage/sealer"
-	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
 	"github.com/filecoin-project/lotus/storage/sealer/sealtasks"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
@@ -285,36 +283,7 @@ var runCmd = &cli.Command{
 			Value:       true,
 			DefaultText: "inherits --addpiece",
 		},
-		&cli.StringFlag{
-			Name:  "external-pc2",
-			Usage: "command for computing PC2 externally",
-		},
 	},
-	Description: `Run lotus-worker.
-
---external-pc2 can be used to compute the PreCommit2 inputs externally.
-The flag behaves similarly to the related lotus-worker flag, using it in
-lotus-bench may be useful for testing if the external PreCommit2 command is
-invoked correctly.
-
-The command will be called with a number of environment variables set:
-* EXTSEAL_PC2_SECTOR_NUM: the sector number
-* EXTSEAL_PC2_SECTOR_MINER: the miner id
-* EXTSEAL_PC2_PROOF_TYPE: the proof type
-* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
-* EXTSEAL_PC2_CACHE: the path to the cache directory
-* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
-* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
-
-The command is expected to:
-* Create cache sc-02-data-tree-r* files
-* Create cache sc-02-data-tree-c* files
-* Create cache p_aux / t_aux files
-* Transform the sealed file in place
-
-Example invocation of lotus-bench as external executor:
-'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
-`,
 	Before: func(cctx *cli.Context) error {
 		if cctx.IsSet("address") {
 			log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'")
@@ -379,18 +348,6 @@ Example invocation of lotus-bench as external executor:
 		// Connect to storage-miner
 		ctx := lcli.ReqContext(cctx)
 
-		// Create a new context with cancel function
-		ctx, cancel := context.WithCancel(ctx)
-		defer cancel()
-
-		// Listen for interrupt signals
-		go func() {
-			c := make(chan os.Signal, 1)
-			signal.Notify(c, os.Interrupt)
-			<-c
-			cancel()
-		}()
-
 		var nodeApi api.StorageMiner
 		var closer func()
 		for {
@@ -402,13 +359,14 @@ Example invocation of lotus-bench as external executor:
 				}
 			}
 			fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err)
-			select {
-			case <-ctx.Done():
-				return xerrors.New("Interrupted by user")
-			case <-time.After(time.Second):
-			}
+			time.Sleep(time.Second)
+			continue
 		}
+
 		defer closer()
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
 		// Register all metric views
 		if err := view.Register(
 			metrics.DefaultViews...,
@@ -653,32 +611,18 @@ Example invocation of lotus-bench as external executor:
 			fh.ServeHTTP(w, r)
 		}
 
-		// Parse ffi executor flags
-
-		var ffiOpts []ffiwrapper.FFIWrapperOpt
-
-		if cctx.IsSet("external-pc2") {
-			extSeal := ffiwrapper.ExternalSealer{
-				PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")),
-			}
-
-			ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal))
-		}
-
 		// Create / expose the worker
 
 		wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix))
 
 		workerApi := &sealworker.Worker{
-			LocalWorker: sealer.NewLocalWorkerWithExecutor(
-				sealer.FFIExec(ffiOpts...),
-				sealer.WorkerConfig{
-					TaskTypes:                 taskTypes,
-					NoSwap:                    cctx.Bool("no-swap"),
-					MaxParallelChallengeReads: cctx.Int("post-parallel-reads"),
-					ChallengeReadTimeout:      cctx.Duration("post-read-timeout"),
-					Name:                      cctx.String("name"),
-				}, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts),
+			LocalWorker: sealer.NewLocalWorker(sealer.WorkerConfig{
+				TaskTypes:                 taskTypes,
+				NoSwap:                    cctx.Bool("no-swap"),
+				MaxParallelChallengeReads: cctx.Int("post-parallel-reads"),
+				ChallengeReadTimeout:      cctx.Duration("post-read-timeout"),
+				Name:                      cctx.String("name"),
+			}, remote, localStore, nodeApi, nodeApi, wsts),
 			LocalStore: localStore,
 			Storage:    lr,
 		}

cmd/lotus/daemon.go

@@ -269,37 +269,7 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		chainfile := cctx.String("import-chain")
-		snapshot := cctx.String("import-snapshot")
-		willImportChain := false
-		if chainfile != "" || snapshot != "" {
-			if chainfile != "" && snapshot != "" {
-				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
-			}
-			willImportChain = true
-		}
-
-		willRemoveChain := cctx.Bool("remove-existing-chain")
-		if willImportChain && !willRemoveChain {
-			// Confirm with the user about the intention to remove chain data.
-			reader := bufio.NewReader(os.Stdin)
-			fmt.Print("Importing chain or snapshot will by default delete existing local chain data. Do you want to proceed and delete? (yes/no): ")
-			userInput, err := reader.ReadString('\n')
-			if err != nil {
-				return xerrors.Errorf("reading user input: %w", err)
-			}
-			userInput = strings.ToLower(strings.TrimSpace(userInput))
-
-			if userInput == "yes" {
-				willRemoveChain = true
-			} else if userInput == "no" {
-				willRemoveChain = false
-			} else {
-				return fmt.Errorf("invalid input. please answer with 'yes' or 'no'")
-			}
-		}
-
-		if willRemoveChain {
+		if cctx.Bool("remove-existing-chain") {
 			lr, err := repo.NewFS(cctx.String("repo"))
 			if err != nil {
 				return xerrors.Errorf("error opening fs repo: %w", err)
@@ -319,7 +289,12 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		if willImportChain {
+		chainfile := cctx.String("import-chain")
+		snapshot := cctx.String("import-snapshot")
+		if chainfile != "" || snapshot != "" {
+			if chainfile != "" && snapshot != "" {
+				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
+			}
 			var issnapshot bool
 			if chainfile == "" {
 				chainfile = snapshot
@@ -565,7 +540,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 	}
 
 	bar.Start()
-	ts, gen, err := cst.Import(ctx, ir)
+	ts, err := cst.Import(ctx, ir)
 	bar.Finish()
 
 	if err != nil {
@@ -576,14 +551,18 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 		return xerrors.Errorf("flushing validation cache failed: %w", err)
 	}
 
-	log.Infof("setting genesis")
-	err = cst.SetGenesis(ctx, gen)
+	gb, err := cst.GetTipsetByHeight(ctx, 0, ts, true)
+	if err != nil {
+		return err
+	}
+
+	err = cst.SetGenesis(ctx, gb.Blocks()[0])
 	if err != nil {
 		return err
 	}
 
 	if !snapshot {
-		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), gen.Timestamp, nil)
+		shd, err := drand.BeaconScheduleFromDrandSchedule(build.DrandConfigSchedule(), gb.MinTimestamp(), nil)
 		if err != nil {
 			return xerrors.Errorf("failed to construct beacon schedule: %w", err)
 		}

documentation/en/api-v0-methods.md

@@ -2289,8 +2289,7 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   },
   [
     {
@@ -2767,8 +2766,7 @@ Inputs:
   ],
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   }
 ]
 ```
@@ -3027,8 +3025,7 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   }
 ]
 ```

documentation/en/api-v1-unstable-methods.md

@@ -2402,7 +2402,14 @@ Perms: read
 Inputs:
 ```json
 [
-  "Bw=="
+  {
+    "from": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+    "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031",
+    "gas": "0x5",
+    "gasPrice": "0x0",
+    "value": "0x0",
+    "data": "0x07"
+  }
 ]
 ```
 
@@ -3350,8 +3357,7 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   },
   [
     {
@@ -3828,8 +3834,7 @@ Inputs:
   ],
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   }
 ]
 ```
@@ -4221,8 +4226,7 @@ Inputs:
   },
   {
     "MaxFee": "0",
-    "MsgUuid": "07070707-0707-0707-0707-070707070707",
-    "MaximizeFeeCap": true
+    "MsgUuid": "07070707-0707-0707-0707-070707070707"
   }
 ]
 ```

documentation/en/block-validation.md

@@ -104,9 +104,9 @@ domain separation tag.
 ### Winning PoSt proof
 
 Draw randomness for current epoch with `WinningPoSt` domain separation tag.
-Get list of sectors challenged in this epoch for this miner, based on the randomness drawn. 
+Get list of sectors challanged in this epoch for this miner, based on the randomness drawn. 
 
-`V`: Use filecoin proofs system to verify that miner proved access to sealed versions of these sectors.
+`V`: Use filecoin proofs system to verify that miner prooved access to sealed versions of these sectors.
 
 ## `(*StateManager).TipSetState()`
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.1
 
 COMMANDS:
    init     Initialize a lotus miner repo

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.1
 
 COMMANDS:
    run        Start lotus worker
@@ -34,33 +34,6 @@ NAME:
 USAGE:
    lotus-worker run [command options] [arguments...]
 
-DESCRIPTION:
-   Run lotus-worker.
-
-   --external-pc2 can be used to compute the PreCommit2 inputs externally.
-   The flag behaves similarly to the related lotus-worker flag, using it in
-   lotus-bench may be useful for testing if the external PreCommit2 command is
-   invoked correctly.
-
-   The command will be called with a number of environment variables set:
-   * EXTSEAL_PC2_SECTOR_NUM: the sector number
-   * EXTSEAL_PC2_SECTOR_MINER: the miner id
-   * EXTSEAL_PC2_PROOF_TYPE: the proof type
-   * EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes
-   * EXTSEAL_PC2_CACHE: the path to the cache directory
-   * EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller)
-   * EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json)
-
-   The command is expected to:
-   * Create cache sc-02-data-tree-r* files
-   * Create cache sc-02-data-tree-c* files
-   * Create cache p_aux / t_aux files
-   * Transform the sealed file in place
-
-   Example invocation of lotus-bench as external executor:
-   './lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT'
-
-
 OPTIONS:
    --listen value                host address and port the worker api will listen on (default: "0.0.0.0:3456") [$LOTUS_WORKER_LISTEN]
    --no-local-storage            don't use storageminer repo for sector storage (default: false) [$LOTUS_WORKER_NO_LOCAL_STORAGE]
@@ -84,7 +57,6 @@ OPTIONS:
    --timeout value               used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function (default: "30m") [$LOTUS_WORKER_TIMEOUT]
    --http-server-timeout value   (default: "30s")
    --data-cid                    Run the data-cid task. true|false (default: inherits --addpiece)
-   --external-pc2 value          command for computing PC2 externally
    --help, -h                    show help
 ```
 

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.25.1-dev
+   1.25.1
 
 COMMANDS:
    daemon   Start a lotus daemon process

documentation/en/default-lotus-config.toml

@@ -195,7 +195,7 @@
 
   [Chainstore.Splitstore]
     # ColdStoreType specifies the type of the coldstore.
-    # It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state..
+    # It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
     #
     # type: string
     # env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE

documentation/en/default-lotus-miner-config.toml

@@ -778,10 +778,6 @@
   # env var: LOTUS_FEES_MAXMARKETBALANCEADDFEE
   #MaxMarketBalanceAddFee = "0.007 FIL"
 
-  # type: bool
-  # env var: LOTUS_FEES_MAXIMIZEWINDOWPOSTFEECAP
-  #MaximizeWindowPoStFeeCap = true
-
   [Fees.MaxPreCommitBatchGasFee]
     # type: types.FIL
     # env var: LOTUS_FEES_MAXPRECOMMITBATCHGASFEE_BASE

documentation/misc/RELEASE_ISSUE_TEMPLATE.md

@@ -14,13 +14,11 @@
 **Note for whoever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases.  There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end.
 
 First steps:
-  - [ ] Set shell variables vX.Y.Z `export X=1 && export Y=24 && export Z=3`
-  - [ ] FFI: Fork a new branch (`release/lotus-vX.Y.Z`) from the filecoin-ffi `master` branch: `git checkout master && git checkout -b release/lotus-v$X.$Y.$Z`
-  - [ ] FFI: Tag the head of `release/lotus-vX.Y.Z` as `vX.Y.Z-pre1`: `git tag -a v$X.$Y.$Z-pre1 -m"release"`
+  - [ ] FFI: Fork a new branch (`release/lotus-vX.Y.Z`) from the filecoin-ffi `master` branch
+  - [ ] FFI: Tag the head of `release/lotus-vX.Y.Z` as `vX.Y.Z-pre1`
   - [ ] Open and land a PR in lotus `master` that updates the FFI dependency to `vX.Y.Z-pre1` as cut in the previous step
   - [ ] Fork a new branch (`release/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage.
-  - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release).
-  - [ ] Run `make gen && make docsgen-cli` before committing changes
+  - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release). Run make gen and make docsgen-cli before committing changes
     
 **Prepping an RC**:
 
@@ -28,7 +26,6 @@ First steps:
 - [ ] run `make gen && make docsgen-cli`
 - [ ] Generate changelog using the script at scripts/mkreleaselog
 - [ ] Add contents of generated text to lotus/CHANGELOG.md in addition to other details
-- [ ] Commit using PR
 - [ ] tag commit with `vX.Y.Z-rcN`
 - [ ] cut a pre-release [here](https://github.com/filecoin-project/lotus/releases/new?prerelease=true)
 

gateway/node.go

@@ -131,7 +131,7 @@ type TargetAPI interface {
 	EthGasPrice(ctx context.Context) (ethtypes.EthBigInt, error)
 	EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthFeeHistory, error)
 	EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt, error)
-	EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error)
+	EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error)
 	EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error)
 	EthSendRawTransaction(ctx context.Context, rawTx ethtypes.EthBytes) (ethtypes.EthHash, error)
 	EthGetLogs(ctx context.Context, filter *ethtypes.EthFilterSpec) (*ethtypes.EthFilterResult, error)

gateway/proxy_eth.go

@@ -353,19 +353,13 @@ func (gw *Node) EthMaxPriorityFeePerGas(ctx context.Context) (ethtypes.EthBigInt
 	return gw.target.EthMaxPriorityFeePerGas(ctx)
 }
 
-func (gw *Node) EthEstimateGas(ctx context.Context, p jsonrpc.RawParams) (ethtypes.EthUint64, error) {
-	// validate params
-	_, err := jsonrpc.DecodeParams[ethtypes.EthEstimateGasParams](p)
-	if err != nil {
-		return ethtypes.EthUint64(0), xerrors.Errorf("decoding params: %w", err)
-	}
-
+func (gw *Node) EthEstimateGas(ctx context.Context, tx ethtypes.EthCall) (ethtypes.EthUint64, error) {
 	if err := gw.limit(ctx, stateRateLimitTokens); err != nil {
 		return 0, err
 	}
 
 	// todo limit gas? to what?
-	return gw.target.EthEstimateGas(ctx, p)
+	return gw.target.EthEstimateGas(ctx, tx)
 }
 
 func (gw *Node) EthCall(ctx context.Context, tx ethtypes.EthCall, blkParam ethtypes.EthBlockNumberOrHash) (ethtypes.EthBytes, error) {

gen/main.go

@@ -92,7 +92,7 @@ func main() {
 	err = gen.WriteTupleEncodersToFile("./chain/exchange/cbor_gen.go", "exchange",
 		exchange.Request{},
 		exchange.Response{},
-		exchange.CompactedMessages{},
+		exchange.CompactedMessagesCBOR{},
 		exchange.BSTipSet{},
 	)
 	if err != nil {

go.mod

@@ -107,7 +107,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
-	github.com/libp2p/go-libp2p v0.31.0
+	github.com/libp2p/go-libp2p v0.30.0
 	github.com/libp2p/go-libp2p-consensus v0.0.1
 	github.com/libp2p/go-libp2p-gorpc v0.5.0
 	github.com/libp2p/go-libp2p-kad-dht v0.24.0
@@ -136,7 +136,6 @@ require (
 	github.com/raulk/go-watchdog v1.3.0
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	github.com/triplewz/poseidon v0.0.0-20220525065023-a7cdb0e183e7
 	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230923211252-36a87e1ba72f
@@ -298,7 +297,7 @@ require (
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.38.1 // indirect
+	github.com/quic-go/quic-go v0.37.6 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
@@ -339,5 +338,3 @@ require (
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
 
 replace github.com/filecoin-project/test-vectors => ./extern/test-vectors
-
-replace github.com/triplewz/poseidon => github.com/magik6k/poseidon v0.0.0-neptune