For log filter criteria with minHeight == maxHeight (one block) resolve the height to a tipsetkeycid before querying the events.db, to exclude any non-canonical results (quick dirty fix, need a proper fix to handle range queries)

docs: Link the release section in the release flow doc

.gitignore

@@ -41,6 +41,7 @@ build/paramfetch.sh
 bin/ipget
 bin/tmp/*
 .idea
+.vscode
 scratchpad
 
 build/builtin-actors/v*

.golangci.yml

@@ -14,6 +14,7 @@ linters:
     - varcheck
     - deadcode
     - scopelint
+    - unused
 
 # We don't want to skip builtin/
 skip-dirs-use-default: false

CHANGELOG.md

@@ -1,10 +1,15 @@
 # Lotus changelog
 
 # UNRELEASED
+- chore: Auto remove local chain data when importing chain file or snapshot ([filecoin-project/lotus#11277](https://github.com/filecoin-project/lotus/pull/11277))
 
 ## New features
 - feat: Added new tracing API (**HIGHLY EXPERIMENTAL**) supporting two RPC methods: `trace_block` and `trace_replayBlockTransactions` ([filecoin-project/lotus#11100](https://github.com/filecoin-project/lotus/pull/11100))
 
+## Improvements
+- fix: Add time slicing to splitstore purging step during compaction to reduce lock congestion [filecoin-project/lotus#11269](https://github.com/filecoin-project/lotus/pull/11269)
+- feat: Added instructions on how to setup Prometheus/Grafana for monitoring a local Lotus node [filecoin-project/lotus#11276](https://github.com/filecoin-project/lotus/pull/11276)
+
 # v1.23.3 / 2023-08-01
 
 This feature release of Lotus includes numerous improvements and enhancements for node operators, ETH RPC-providers and storage providers.

LOTUS_RELEASE_FLOW.md

@@ -73,7 +73,7 @@ All releases under an odd minor version number indicate **feature releases**. Th
 
 Feature releases include new development and bug fixes. They are not mandatory, but still highly recommended, **as they may contain critical security fixes**. Note that some of these releases may be very small patch releases that include critical hotfixes.  There is no way to distinguish between a bug fix release and a feature release on the "feature" version. Both cases will use the "patch" version number.
 
-We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the Release Cycle section (TODO: Link).
+We aim to ship a new feature release of the Lotus software from our development (master) branch every 3 weeks, so users can expect a regular cadence of Lotus feature releases. Note that mandatory releases for network upgrades may disrupt this schedule. For more, see the [Release Cycle section](#release-cycle).
 
 ### Examples Scenarios
 

README.md

@@ -133,6 +133,8 @@ Note: The default branch `master` is the dev branch where the latest new feature
 
 6. You should now have Lotus installed. You can now [start the Lotus daemon and sync the chain](https://lotus.filecoin.io/lotus/install/linux/#start-the-lotus-daemon-and-sync-the-chain).
 
+7. (Optional) Follow the [Setting Up Prometheus and Grafana](https://github.com/filecoin-project/lotus/tree/master/metrics/README.md) guide for detailed instructions on setting up a working monitoring system running against a local running lotus node.
+
 ## License
 
 Dual-licensed under [MIT](https://github.com/filecoin-project/lotus/blob/master/LICENSE-MIT) + [Apache 2.0](https://github.com/filecoin-project/lotus/blob/master/LICENSE-APACHE)

blockstore/splitstore/splitstore.go

@@ -182,7 +182,6 @@ type SplitStore struct {
 
 	compactionIndex int64
 	pruneIndex      int64
-	onlineGCCnt     int64
 
 	ctx    context.Context
 	cancel func()

blockstore/splitstore/splitstore_compact.go

@@ -66,8 +66,9 @@ var (
 )
 
 const (
-	batchSize  = 16384
-	cidKeySize = 128
+	batchSize              = 16384
+	cidKeySize             = 128
+	purgeWorkSliceDuration = time.Second
 )
 
 func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error {
@@ -1372,9 +1373,21 @@ func (s *SplitStore) purge(coldr *ColdSetReader, checkpoint *Checkpoint, markSet
 		return err
 	}
 
+	now := time.Now()
+
 	err := coldr.ForEach(func(c cid.Cid) error {
 		batch = append(batch, c)
 		if len(batch) == batchSize {
+			// add some time slicing to the purge as this a very disk I/O heavy operation that
+			// requires write access to txnLk that may starve other operations that require
+			// access to the blockstore.
+			elapsed := time.Since(now)
+			if elapsed > purgeWorkSliceDuration {
+				// work 1 slice, sleep 4 slices, or 20% utilization
+				time.Sleep(4 * elapsed)
+				now = time.Now()
+			}
+
 			return deleteBatch()
 		}
 

build/bootstrap/mainnet.pi

@@ -13,4 +13,5 @@
 /dns4/node.glif.io/tcp/1235/p2p/12D3KooWBF8cpp65hp2u9LK5mh19x67ftAam84z9LsfaquTDSBpt
 /dns4/bootstrap-0.ipfsmain.cn/tcp/34721/p2p/12D3KooWQnwEGNqcM2nAcPtRR9rAX8Hrg4k9kJLCHoTR5chJfz6d
 /dns4/bootstrap-1.ipfsmain.cn/tcp/34723/p2p/12D3KooWMKxMkD5DMpSWsW7dBddKxKT7L2GgbNuckz9otxvkvByP
-/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
+/dns4/bootstarp-0.1475.io/tcp/61256/p2p/12D3KooWRzCVDwHUkgdK7eRgnoXbjDAELhxPErjHzbRLguSV1aRt
+/dns4/bootstrap-venus.mainnet.filincubator.com/tcp/8888/p2p/QmQu8C6deXwKvJP2D8B6QGyhngc3ZiDnFzEHBDx8yeBXST

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.23.4-dev"
+const BuildVersion = "1.23.5-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/beacon/drand/drand.go

@@ -29,19 +29,6 @@ import (
 
 var log = logging.Logger("drand")
 
-type drandPeer struct {
-	addr string
-	tls  bool
-}
-
-func (dp *drandPeer) Address() string {
-	return dp.addr
-}
-
-func (dp *drandPeer) IsTLS() bool {
-	return dp.tls
-}
-
 // DrandBeacon connects Lotus with a drand network in order to provide
 // randomness to the system in a way that's aligned with Filecoin rounds/epochs.
 //

chain/consensus/compute_state.go

@@ -147,9 +147,6 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
 				return xerrors.Errorf("callback failed on cron message: %w", err)
 			}
 		}
-		if ret.ExitCode != 0 {
-			return xerrors.Errorf("cron exit was non-zero: %d", ret.ExitCode)
-		}
 
 		return nil
 	}

chain/consensus/filcns/filecoin.go

@@ -91,9 +91,6 @@ var RewardFunc = func(ctx context.Context, vmi vm.Interface, em stmgr.ExecMonito
 		}
 	}
 
-	if ret.ExitCode != 0 {
-		return xerrors.Errorf("reward application message failed (exit %d): %s", ret.ExitCode, ret.ActorErr)
-	}
 	return nil
 }
 

chain/events/filter/event.go

@@ -387,6 +387,19 @@ func (m *EventFilterManager) Install(ctx context.Context, minHeight, maxHeight a
 	}
 
 	if m.EventIndex != nil && minHeight != -1 && minHeight < currentHeight {
+		// hacky fix to fix issue with non-canonical log results being returned
+		// only works when the filter has the same start and stop height
+		if minHeight == maxHeight {
+			canonicalTipSet, err := m.ChainStore.GetTipsetByHeight(ctx, minHeight, nil, false)
+			if err != nil {
+				return nil, err
+			}
+			canonicalTipSetCID, err := canonicalTipSet.Key().Cid()
+			if err != nil {
+				return nil, err
+			}
+			f.tipsetCid = canonicalTipSetCID
+		}
 		// Filter needs historic events
 		if err := m.EventIndex.PrefillFilter(ctx, f); err != nil {
 			return nil, err

chain/events/state/ctxstore.go

@@ -1,25 +0,0 @@
-package state
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	cbor "github.com/ipfs/go-ipld-cbor"
-)
-
-type contextStore struct {
-	ctx context.Context
-	cst *cbor.BasicIpldStore
-}
-
-func (cs *contextStore) Context() context.Context {
-	return cs.ctx
-}
-
-func (cs *contextStore) Get(ctx context.Context, c cid.Cid, out interface{}) error {
-	return cs.cst.Get(ctx, c, out)
-}
-
-func (cs *contextStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) {
-	return cs.cst.Put(ctx, v)
-}

chain/exchange/client.go

@@ -381,7 +381,7 @@ func (c *client) sendRequestToPeer(ctx context.Context, peer peer.ID, req *Reque
 	defer span.End()
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
-			trace.StringAttribute("peer", peer.Pretty()),
+			trace.StringAttribute("peer", peer.String()),
 		)
 	}
 	defer func() {

chain/market/store.go

@@ -39,23 +39,6 @@ func (ps *Store) save(ctx context.Context, state *FundedAddressState) error {
 	return ps.ds.Put(ctx, k, b)
 }
 
-// get the state for the given address
-func (ps *Store) get(ctx context.Context, addr address.Address) (*FundedAddressState, error) {
-	k := dskeyForAddr(addr)
-
-	data, err := ps.ds.Get(ctx, k)
-	if err != nil {
-		return nil, err
-	}
-
-	var state FundedAddressState
-	err = cborrpc.ReadCborRPC(bytes.NewReader(data), &state)
-	if err != nil {
-		return nil, err
-	}
-	return &state, nil
-}
-
 // forEach calls iter with each address in the datastore
 func (ps *Store) forEach(ctx context.Context, iter func(*FundedAddressState)) error {
 	res, err := ps.ds.Query(ctx, dsq.Query{Prefix: dsKeyAddr})

chain/stmgr/forks.go

@@ -232,11 +232,6 @@ func (sm *StateManager) hasExpensiveForkBetween(parent, height abi.ChainEpoch) b
 	return false
 }
 
-func (sm *StateManager) hasExpensiveFork(height abi.ChainEpoch) bool {
-	_, ok := sm.expensiveUpgrades[height]
-	return ok
-}
-
 func runPreMigration(ctx context.Context, sm *StateManager, fn PreMigrationFunc, cache *nv16.MemMigrationCache, ts *types.TipSet) {
 	height := ts.Height()
 	parent := ts.ParentState()

chain/sub/ratelimit/queue_test.go

@@ -0,0 +1,61 @@
+package ratelimit
+
+import (
+	"testing"
+)
+
+func TestQueue(t *testing.T) {
+	const size = 3
+	q := &queue{buf: make([]int64, size)}
+
+	if q.len() != 0 {
+		t.Fatalf("q.len() = %d, expect 0", q.len())
+	}
+
+	if q.cap() != size {
+		t.Fatalf("q.cap() = %d, expect %d", q.cap(), size)
+	}
+
+	for i := int64(0); i < int64(size); i++ {
+		err := q.push(i)
+		if err != nil {
+			t.Fatalf("cannot push element %d", i)
+		}
+	}
+
+	if q.len() != size {
+		t.Fatalf("q.len() = %d, expect %d", q.len(), size)
+	}
+
+	err := q.push(int64(size))
+	if err != ErrRateLimitExceeded {
+		t.Fatalf("pushing element beyond capacity should have failed with err: %s, got %s", ErrRateLimitExceeded, err)
+	}
+
+	if q.front() != 0 {
+		t.Fatalf("q.front() = %d, expect 0", q.front())
+	}
+
+	if q.back() != int64(size-1) {
+		t.Fatalf("q.back() = %d, expect %d", q.back(), size-1)
+	}
+
+	popVal := q.pop()
+	if popVal != 0 {
+		t.Fatalf("q.pop() = %d, expect 0", popVal)
+	}
+
+	if q.len() != size-1 {
+		t.Fatalf("q.len() = %d, expect %d", q.len(), size-1)
+	}
+
+	// Testing truncation.
+	threshold := int64(1)
+	q.truncate(threshold)
+	if q.len() != 1 {
+		t.Fatalf("q.len() after truncate = %d, expect 1", q.len())
+	}
+	if q.front() != 2 {
+		t.Fatalf("q.front() after truncate = %d, expect 2", q.front())
+	}
+}

chain/sync_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/libp2p/go-libp2p/core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
 
@@ -344,13 +343,6 @@ func (tu *syncTestUtil) addClientNode() int {
 	return len(tu.nds) - 1
 }
 
-func (tu *syncTestUtil) pid(n int) peer.ID {
-	nal, err := tu.nds[n].NetAddrsListen(tu.ctx)
-	require.NoError(tu.t, err)
-
-	return nal.ID
-}
-
 func (tu *syncTestUtil) connect(from, to int) {
 	toPI, err := tu.nds[to].NetAddrsListen(tu.ctx)
 	require.NoError(tu.t, err)

chain/types/blockmsg_test.go

@@ -7,9 +7,6 @@ import (
 )
 
 func TestDecodeBlockMsg(t *testing.T) {
-	type args struct {
-		b []byte
-	}
 	tests := []struct {
 		name    string
 		data    []byte

chain/types/vmcontext.go

@@ -27,24 +27,3 @@ type StateTree interface {
 
 	Version() StateTreeVersion
 }
-
-type storageWrapper struct {
-	s Storage
-}
-
-func (sw *storageWrapper) Put(i cbg.CBORMarshaler) (cid.Cid, error) {
-	c, err := sw.s.Put(i)
-	if err != nil {
-		return cid.Undef, err
-	}
-
-	return c, nil
-}
-
-func (sw *storageWrapper) Get(c cid.Cid, out cbg.CBORUnmarshaler) error {
-	if err := sw.s.Get(c, out); err != nil {
-		return err
-	}
-
-	return nil
-}

cli/net.go

@@ -282,7 +282,7 @@ var NetDisconnect = &cli.Command{
 				fmt.Println("failure")
 				return err
 			}
-			fmt.Printf("disconnect %s: ", pid.Pretty())
+			fmt.Printf("disconnect %s: ", pid)
 			err = api.NetDisconnect(ctx, pid)
 			if err != nil {
 				fmt.Println("failure")
@@ -312,7 +312,7 @@ var NetConnect = &cli.Command{
 		}
 
 		for _, pi := range pis {
-			fmt.Printf("connect %s: ", pi.ID.Pretty())
+			fmt.Printf("connect %s: ", pi.ID)
 			err := api.NetConnect(ctx, pi)
 			if err != nil {
 				fmt.Println("failure")

cli/sync.go

@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 )
 
 var SyncCmd = &cli.Command{
@@ -262,6 +263,9 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 	}
 	firstApp = state.VMApplied
 
+	// eta computes the ETA for the sync to complete (with a lookback of 10 processed items)
+	eta := cliutil.NewETA(10)
+
 	for {
 		state, err := napi.SyncState(ctx)
 		if err != nil {
@@ -312,8 +316,10 @@ func SyncWait(ctx context.Context, napi v0api.FullNode, watch bool) error {
 			fmt.Print("\r\x1b[2K\x1b[A")
 		}
 
+		todo := theight - ss.Height
+
 		fmt.Printf("Worker: %d; Base: %d; Target: %d (diff: %d)\n", workerID, baseHeight, theight, heightDiff)
-		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d\n", ss.Stage, ss.Height, theight-ss.Height)
+		fmt.Printf("State: %s; Current Epoch: %d; Todo: %d, ETA: %s\n", ss.Stage, ss.Height, todo, eta.Update(int64(todo)))
 		lastLines = 2
 
 		if i%samples == 0 {

cli/util/eta.go

@@ -0,0 +1,94 @@
+package cliutil
+
+import (
+	"fmt"
+	"math"
+	"time"
+)
+
+// ETA implements a very simple eta calculator based on the number of remaining items. It does not
+// require knowing the work size in advance and is therefore suitable for streaming workloads and
+// also does not require that consecutive updates have a monotonically decreasing remaining value.
+type ETA struct {
+	// max number of items to keep in memory
+	maxItems int
+	// a queue of most recently updated items
+	items []item
+	// we store the last calculated ETA which we reuse if there was not change in remaining items
+	lastETA string
+}
+
+type item struct {
+	timestamp time.Time
+	remaining int64
+}
+
+// NewETA creates a new ETA calculator of the given size
+func NewETA(maxItems int) *ETA {
+	return &ETA{
+		maxItems: maxItems,
+		items:    make([]item, 0),
+	}
+}
+
+// Update updates the ETA calculator with the remaining number of items and returns the ETA
+func (e *ETA) Update(remaining int64) string {
+	item := item{
+		timestamp: time.Now(),
+		remaining: remaining,
+	}
+
+	if len(e.items) == 0 {
+		e.items = append(e.items, item)
+		return ""
+	}
+
+	if e.items[len(e.items)-1].remaining == remaining {
+		// we ignore updates with the same remaining value and just return the previous ETA
+		return e.lastETA
+	} else if e.items[len(e.items)-1].remaining < remaining {
+		// remaining went up from previous update, lets estimate how many items were processed using the
+		// average number processed items in the queue.
+		var avgProcessedPerItem int64 = 1
+		if len(e.items) > 1 {
+			diffRemaining := e.items[0].remaining - e.items[len(e.items)-1].remaining
+			avgProcessedPerItem = int64(math.Round(float64(diffRemaining) / float64(len(e.items))))
+		}
+
+		// diff is the difference in increase in remaining since last update plus the average number of processed
+		// items we estimate that were processed this round
+		diff := remaining - e.items[len(e.items)-1].remaining + avgProcessedPerItem
+
+		// we update all items in the queue by shifting their remaining value accordingly. This means that we
+		// always have strictly decreasing remaining values in the queue
+		for i := range e.items {
+			e.items[i].remaining += diff
+		}
+	}
+
+	// append the item to the queue and remove the oldest item if needed
+	if len(e.items) >= e.maxItems {
+		e.items = e.items[1:]
+	}
+	e.items = append(e.items, item)
+
+	// calculate the average processing time per item in the queue
+	diffMs := e.items[len(e.items)-1].timestamp.Sub(e.items[0].timestamp).Milliseconds()
+	nrItemsProcessed := e.items[0].remaining - e.items[len(e.items)-1].remaining
+	avg := diffMs / nrItemsProcessed
+
+	// use that average processing time to estimate how long the remaining items will take
+	// and cache that ETA so we don't have to recalculate it on every update unless the
+	// remaining value changes
+	e.lastETA = msToETA(avg * remaining)
+
+	return e.lastETA
+}
+
+func msToETA(ms int64) string {
+	seconds := ms / 1000
+	minutes := seconds / 60
+	hours := minutes / 60
+
+	return fmt.Sprintf("%02dh:%02dm:%02ds", hours, minutes%60, seconds%60)
+}

cli/wallet.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/tablewriter"
 )
@@ -459,7 +460,12 @@ var walletSign = &cli.Command{
 		sig, err := api.WalletSign(ctx, addr, msg)
 
 		if err != nil {
-			return err
+			// Check if the address is a multisig address
+			act, actErr := api.StateGetActor(ctx, addr, types.EmptyTSK)
+			if actErr == nil && builtin.IsMultisigActor(act.Code) {
+				return xerrors.Errorf("specified signer address is a multisig actor, it doesn’t have keys to sign transactions. To send a message with a multisig, signers of the multisig need to propose and approve transactions.")
+			}
+			return xerrors.Errorf("failed to sign message: %w", err)
 		}
 
 		sigBytes := append([]byte{byte(sig.Type)}, sig.Data...)

cmd/lotus-bench/cli.go

@@ -0,0 +1,312 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cliCmd = &cli.Command{
+	Name:  "cli",
+	Usage: "Runs a concurrent stress test on one or more binaries commands and prints the performance metrics including latency distribution and histogram",
+	Description: `This benchmark has the following features:
+* Can query each command both sequentially and concurrently
+* Supports rate limiting
+* Can query multiple different commands at once (supporting different concurrency level and rate limiting for each command)
+* Gives a nice reporting summary of the stress testing of each command (including latency distribution, histogram and more)
+* Easy to use
+
+To use this benchmark you must specify the commands you want to test using the --cmd options, the format of it is:
+
+  --cmd=CMD[:CONCURRENCY][:QPS] where only NAME is required.
+
+Here are some real examples:
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' // runs the command with default concurrency and qps
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3'  // override concurrency to 3
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages::100' // override to 100 qps while using default concurrency
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages:3:100' // run using 3 workers but limit to 100 qps
+  lotus-bench cli --cmd='lotus-shed mpool miner-select-messages' --cmd='lotus sync wait' // run two commands at once
+`,
+	Flags: []cli.Flag{
+		&cli.DurationFlag{
+			Name:  "duration",
+			Value: 60 * time.Second,
+			Usage: "Duration of benchmark in seconds",
+		},
+		&cli.IntFlag{
+			Name:  "concurrency",
+			Value: 10,
+			Usage: "How many workers should be used per command (can be overridden per command)",
+		},
+		&cli.IntFlag{
+			Name:  "qps",
+			Value: 0,
+			Usage: "How many requests per second should be sent per command (can be overridden per command), a value of 0 means no limit",
+		},
+		&cli.StringSliceFlag{
+			Name:  "cmd",
+			Usage: `Command to benchmark, you can specify multiple commands by repeating this flag. You can also specify command specific options to set the concurrency and qps for each command (see usage).`,
+		},
+		&cli.DurationFlag{
+			Name:  "watch",
+			Value: 0 * time.Second,
+			Usage: "If >0 then generates reports every N seconds (only supports linux/unix)",
+		},
+		&cli.BoolFlag{
+			Name:  "print-response",
+			Value: false,
+			Usage: "print the response of each request",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if len(cctx.StringSlice("cmd")) == 0 {
+			return errors.New("you must specify and least one cmd to benchmark")
+		}
+
+		var cmds []*CMD
+		for _, str := range cctx.StringSlice("cmd") {
+			entries := strings.SplitN(str, ":", 3)
+			if len(entries) == 0 {
+				return errors.New("invalid cmd format")
+			}
+
+			// check if concurrency was specified
+			concurrency := cctx.Int("concurrency")
+			if len(entries) > 1 {
+				if len(entries[1]) > 0 {
+					var err error
+					concurrency, err = strconv.Atoi(entries[1])
+					if err != nil {
+						return fmt.Errorf("could not parse concurrency value from command %s: %v", entries[0], err)
+					}
+				}
+			}
+
+			// check if qps was specified
+			qps := cctx.Int("qps")
+			if len(entries) > 2 {
+				if len(entries[2]) > 0 {
+					var err error
+					qps, err = strconv.Atoi(entries[2])
+					if err != nil {
+						return fmt.Errorf("could not parse qps value from command %s: %v", entries[0], err)
+					}
+				}
+			}
+
+			cmds = append(cmds, &CMD{
+				w:           os.Stdout,
+				cmd:         entries[0],
+				concurrency: concurrency,
+				qps:         qps,
+				printResp:   cctx.Bool("print-response"),
+			})
+		}
+
+		// terminate early on ctrl+c
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+		go func() {
+			<-c
+			fmt.Println("Received interrupt, stopping...")
+			for _, cmd := range cmds {
+				cmd.Stop()
+			}
+		}()
+
+		// stop all threads after duration
+		go func() {
+			time.Sleep(cctx.Duration("duration"))
+			for _, cmd := range cmds {
+				cmd.Stop()
+			}
+		}()
+
+		// start all threads
+		var wg sync.WaitGroup
+		wg.Add(len(cmds))
+
+		for _, cmd := range cmds {
+			go func(cmd *CMD) {
+				defer wg.Done()
+				err := cmd.Run()
+				if err != nil {
+					fmt.Printf("error running cmd: %v\n", err)
+				}
+			}(cmd)
+		}
+
+		// if watch is set then print a report every N seconds
+		var progressCh chan struct{}
+		if cctx.Duration("watch") > 0 {
+			progressCh = make(chan struct{}, 1)
+			go func(progressCh chan struct{}) {
+				ticker := time.NewTicker(cctx.Duration("watch"))
+				for {
+					clearAndPrintReport := func() {
+						// clear the screen move the cursor to the top left
+						fmt.Print("\033[2J")
+						fmt.Printf("\033[%d;%dH", 1, 1)
+						for i, cmd := range cmds {
+							cmd.Report()
+							if i < len(cmds)-1 {
+								fmt.Println()
+							}
+						}
+					}
+					select {
+					case <-ticker.C:
+						clearAndPrintReport()
+					case <-progressCh:
+						clearAndPrintReport()
+						return
+					}
+				}
+			}(progressCh)
+		}
+
+		wg.Wait()
+
+		if progressCh != nil {
+			// wait for the watch go routine to return
+			progressCh <- struct{}{}
+
+			// no need to print the report again
+			return nil
+		}
+
+		// print the report for each command
+		for i, cmd := range cmds {
+			cmd.Report()
+			if i < len(cmds)-1 {
+				fmt.Println()
+			}
+		}
+
+		return nil
+	},
+}
+
+// CMD handles the benchmarking of a single command.
+type CMD struct {
+	w io.Writer
+	// the cmd we want to benchmark
+	cmd string
+	// the number of concurrent requests to make to this command
+	concurrency int
+	// if >0 then limit to qps is the max number of requests per second to make to this command (0 = no limit)
+	qps int
+	// whether or not to print the response of each request (useful for debugging)
+	printResp bool
+	// instruct the worker go routines to stop
+	stopCh chan struct{}
+	// when the command bencharking started
+	start time.Time
+	// results channel is used by the workers to send results to the reporter
+	results chan *result
+	// reporter handles reading the results from workers and printing the report statistics
+	reporter *Reporter
+}
+
+func (c *CMD) Run() error {
+	var wg sync.WaitGroup
+	wg.Add(c.concurrency)
+
+	c.results = make(chan *result, c.concurrency*1_000)
+	c.stopCh = make(chan struct{}, c.concurrency)
+
+	go func() {
+		c.reporter = NewReporter(c.results, c.w)
+		c.reporter.Run()
+	}()
+
+	c.start = time.Now()
+
+	// throttle the number of requests per second
+	var qpsTicker *time.Ticker
+	if c.qps > 0 {
+		qpsTicker = time.NewTicker(time.Second / time.Duration(c.qps))
+	}
+
+	for i := 0; i < c.concurrency; i++ {
+		go func() {
+			c.startWorker(qpsTicker)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	// close the results channel so reporter will stop
+	close(c.results)
+
+	// wait until the reporter is done
+	<-c.reporter.doneCh
+
+	return nil
+}
+
+func (c *CMD) startWorker(qpsTicker *time.Ticker) {
+	for {
+		// check if we should stop
+		select {
+		case <-c.stopCh:
+			return
+		default:
+		}
+
+		// wait for the next tick if we are rate limiting this command
+		if qpsTicker != nil {
+			<-qpsTicker.C
+		}
+
+		start := time.Now()
+
+		var statusCode int = 0
+
+		arr := strings.Fields(c.cmd)
+
+		data, err := exec.Command(arr[0], arr[1:]...).Output()
+		if err != nil {
+			fmt.Println("1")
+			if exitError, ok := err.(*exec.ExitError); ok {
+				statusCode = exitError.ExitCode()
+			} else {
+				statusCode = 1
+			}
+		} else {
+			if c.printResp {
+				fmt.Printf("[%s] %s", c.cmd, string(data))
+			}
+		}
+
+		c.results <- &result{
+			statusCode: &statusCode,
+			err:        err,
+			duration:   time.Since(start),
+		}
+	}
+}
+
+func (c *CMD) Stop() {
+	for i := 0; i < c.concurrency; i++ {
+		c.stopCh <- struct{}{}
+	}
+}
+
+func (c *CMD) Report() {
+	total := time.Since(c.start)
+	fmt.Fprintf(c.w, "[%s]:\n", c.cmd)
+	fmt.Fprintf(c.w, "- Options:\n")
+	fmt.Fprintf(c.w, "  - concurrency: %d\n", c.concurrency)
+	fmt.Fprintf(c.w, "  - qps: %d\n", c.qps)
+	c.reporter.Print(total, c.w)
+}

cmd/lotus-bench/import.go

@@ -497,21 +497,6 @@ type Invocation struct {
 
 const GasPerNs = 10
 
-func countGasCosts(et *types.ExecutionTrace) int64 {
-	var cgas int64
-
-	for _, gc := range et.GasCharges {
-		cgas += gc.ComputeGas
-	}
-
-	for _, sub := range et.Subcalls {
-		c := countGasCosts(&sub) //nolint
-		cgas += c
-	}
-
-	return cgas
-}
-
 type stats struct {
 	timeTaken meanVar
 	gasRatio  meanVar

cmd/lotus-bench/main.go

@@ -120,6 +120,7 @@ func main() {
 			sealBenchCmd,
 			simpleCmd,
 			importBenchCmd,
+			cliCmd,
 			rpcCmd,
 		},
 	}

cmd/lotus-bench/reporter.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+)
+
+// result is the result of a single rpc method request.
+type result struct {
+	err        error
+	statusCode *int
+	duration   time.Duration
+}
+
+// Reporter reads the results from the workers through the results channel and aggregates the results.
+type Reporter struct {
+	// write the report to this writer
+	w io.Writer
+	// the reporter read the results from this channel
+	results chan *result
+	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
+	doneCh chan bool
+
+	// lock protect the following fields during critical sections (if --watch was specified)
+	lock sync.Mutex
+	// the latencies of all requests
+	latencies []int64
+	// the number of requests that returned each status code
+	statusCodes map[int]int
+	// the number of errors that occurred
+	errors map[string]int
+}
+
+func NewReporter(results chan *result, w io.Writer) *Reporter {
+	return &Reporter{
+		w:           w,
+		results:     results,
+		doneCh:      make(chan bool, 1),
+		statusCodes: make(map[int]int),
+		errors:      make(map[string]int),
+	}
+}
+
+func (r *Reporter) Run() {
+	for res := range r.results {
+		r.lock.Lock()
+
+		r.latencies = append(r.latencies, res.duration.Milliseconds())
+
+		if res.statusCode != nil {
+			r.statusCodes[*res.statusCode]++
+		}
+
+		if res.err != nil {
+			if len(r.errors) < 1_000_000 {
+				r.errors[res.err.Error()]++
+			} else {
+				// we don't want to store too many errors in memory
+				r.errors["hidden"]++
+			}
+		} else {
+			r.errors["nil"]++
+		}
+
+		r.lock.Unlock()
+	}
+
+	r.doneCh <- true
+}
+
+func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	nrReq := int64(len(r.latencies))
+	if nrReq == 0 {
+		fmt.Println("No requests were made")
+		return
+	}
+
+	// we need to sort the latencies slice to calculate the percentiles
+	sort.Slice(r.latencies, func(i, j int) bool {
+		return r.latencies[i] < r.latencies[j]
+	})
+
+	var totalLatency int64 = 0
+	for _, latency := range r.latencies {
+		totalLatency += latency
+	}
+
+	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
+	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
+	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
+	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
+	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
+	fmt.Fprintf(w, "- Latency distribution:\n")
+	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
+	for _, p := range percentiles {
+		idx := int64(p * float64(nrReq))
+		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
+	}
+
+	// create a simple histogram with 10 buckets spanning the range of latency
+	// into equal ranges
+	//
+	nrBucket := 10
+	buckets := make([]Bucket, nrBucket)
+	latencyRange := r.latencies[len(r.latencies)-1]
+	bucketRange := latencyRange / int64(nrBucket)
+
+	// mark the end of each bucket
+	for i := 0; i < nrBucket; i++ {
+		buckets[i].start = int64(i) * bucketRange
+		buckets[i].end = buckets[i].start + bucketRange
+		// extend the last bucked by any remaning range caused by the integer division
+		if i == nrBucket-1 {
+			buckets[i].end = latencyRange
+		}
+	}
+
+	// count the number of requests in each bucket
+	currBucket := 0
+	for i := 0; i < len(r.latencies); {
+		if r.latencies[i] <= buckets[currBucket].end {
+			buckets[currBucket].cnt++
+			i++
+		} else {
+			currBucket++
+		}
+	}
+
+	// print the histogram using a tabwriter which will align the columns nicely
+	fmt.Fprintf(w, "- Histogram:\n")
+	const padding = 2
+	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
+	for i := 0; i < nrBucket; i++ {
+		ratio := float64(buckets[i].cnt) / float64(nrReq)
+		bars := strings.Repeat("#", int(ratio*100))
+		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
+	}
+	tabWriter.Flush() //nolint:errcheck
+
+	fmt.Fprintf(w, "- Status codes:\n")
+	for code, cnt := range r.statusCodes {
+		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
+	}
+
+	// print the 10 most occurring errors (in case error values are not unique)
+	//
+	type kv struct {
+		err string
+		cnt int
+	}
+	var sortedErrors []kv
+	for err, cnt := range r.errors {
+		sortedErrors = append(sortedErrors, kv{err, cnt})
+	}
+	sort.Slice(sortedErrors, func(i, j int) bool {
+		return sortedErrors[i].cnt > sortedErrors[j].cnt
+	})
+	fmt.Fprintf(w, "- Errors (top 10):\n")
+	for i, se := range sortedErrors {
+		if i > 10 {
+			break
+		}
+		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
+	}
+}
+
+type Bucket struct {
+	start int64
+	// the end value of the bucket
+	end int64
+	// how many entries are in the bucket
+	cnt int
+}

cmd/lotus-bench/rpc.go

@@ -9,11 +9,9 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
-	"text/tabwriter"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -243,13 +241,6 @@ type RPCMethod struct {
 	reporter *Reporter
 }
 
-// result is the result of a single rpc method request.
-type result struct {
-	err        error
-	statusCode *int
-	duration   time.Duration
-}
-
 func (rpc *RPCMethod) Run() error {
 	client := &http.Client{
 		Timeout: 0,
@@ -411,166 +402,3 @@ func (rpc *RPCMethod) Report() {
 	fmt.Fprintf(rpc.w, "  - qps: %d\n", rpc.qps)
 	rpc.reporter.Print(total, rpc.w)
 }
-
-// Reporter reads the results from the workers through the results channel and aggregates the results.
-type Reporter struct {
-	// write the report to this writer
-	w io.Writer
-	// the reporter read the results from this channel
-	results chan *result
-	// doneCh is used to signal that the reporter has finished reading the results (channel has closed)
-	doneCh chan bool
-
-	// lock protect the following fields during critical sections (if --watch was specified)
-	lock sync.Mutex
-	// the latencies of all requests
-	latencies []int64
-	// the number of requests that returned each status code
-	statusCodes map[int]int
-	// the number of errors that occurred
-	errors map[string]int
-}
-
-func NewReporter(results chan *result, w io.Writer) *Reporter {
-	return &Reporter{
-		w:           w,
-		results:     results,
-		doneCh:      make(chan bool, 1),
-		statusCodes: make(map[int]int),
-		errors:      make(map[string]int),
-	}
-}
-
-func (r *Reporter) Run() {
-	for res := range r.results {
-		r.lock.Lock()
-
-		r.latencies = append(r.latencies, res.duration.Milliseconds())
-
-		if res.statusCode != nil {
-			r.statusCodes[*res.statusCode]++
-		}
-
-		if res.err != nil {
-			if len(r.errors) < 1_000_000 {
-				r.errors[res.err.Error()]++
-			} else {
-				// we don't want to store too many errors in memory
-				r.errors["hidden"]++
-			}
-		} else {
-			r.errors["nil"]++
-		}
-
-		r.lock.Unlock()
-	}
-
-	r.doneCh <- true
-}
-
-func (r *Reporter) Print(elapsed time.Duration, w io.Writer) {
-	r.lock.Lock()
-	defer r.lock.Unlock()
-
-	nrReq := int64(len(r.latencies))
-	if nrReq == 0 {
-		fmt.Println("No requests were made")
-		return
-	}
-
-	// we need to sort the latencies slice to calculate the percentiles
-	sort.Slice(r.latencies, func(i, j int) bool {
-		return r.latencies[i] < r.latencies[j]
-	})
-
-	var totalLatency int64 = 0
-	for _, latency := range r.latencies {
-		totalLatency += latency
-	}
-
-	fmt.Fprintf(w, "- Total Requests: %d\n", nrReq)
-	fmt.Fprintf(w, "- Total Duration: %dms\n", elapsed.Milliseconds())
-	fmt.Fprintf(w, "- Requests/sec: %f\n", float64(nrReq)/elapsed.Seconds())
-	fmt.Fprintf(w, "- Avg latency: %dms\n", totalLatency/nrReq)
-	fmt.Fprintf(w, "- Median latency: %dms\n", r.latencies[nrReq/2])
-	fmt.Fprintf(w, "- Latency distribution:\n")
-	percentiles := []float64{0.1, 0.5, 0.9, 0.95, 0.99, 0.999}
-	for _, p := range percentiles {
-		idx := int64(p * float64(nrReq))
-		fmt.Fprintf(w, "    %s%% in %dms\n", fmt.Sprintf("%.2f", p*100.0), r.latencies[idx])
-	}
-
-	// create a simple histogram with 10 buckets spanning the range of latency
-	// into equal ranges
-	//
-	nrBucket := 10
-	buckets := make([]Bucket, nrBucket)
-	latencyRange := r.latencies[len(r.latencies)-1]
-	bucketRange := latencyRange / int64(nrBucket)
-
-	// mark the end of each bucket
-	for i := 0; i < nrBucket; i++ {
-		buckets[i].start = int64(i) * bucketRange
-		buckets[i].end = buckets[i].start + bucketRange
-		// extend the last bucked by any remaning range caused by the integer division
-		if i == nrBucket-1 {
-			buckets[i].end = latencyRange
-		}
-	}
-
-	// count the number of requests in each bucket
-	currBucket := 0
-	for i := 0; i < len(r.latencies); {
-		if r.latencies[i] <= buckets[currBucket].end {
-			buckets[currBucket].cnt++
-			i++
-		} else {
-			currBucket++
-		}
-	}
-
-	// print the histogram using a tabwriter which will align the columns nicely
-	fmt.Fprintf(w, "- Histogram:\n")
-	const padding = 2
-	tabWriter := tabwriter.NewWriter(w, 0, 0, padding, ' ', tabwriter.AlignRight|tabwriter.Debug)
-	for i := 0; i < nrBucket; i++ {
-		ratio := float64(buckets[i].cnt) / float64(nrReq)
-		bars := strings.Repeat("#", int(ratio*100))
-		fmt.Fprintf(tabWriter, "  %d-%dms\t%d\t%s (%s%%)\n", buckets[i].start, buckets[i].end, buckets[i].cnt, bars, fmt.Sprintf("%.2f", ratio*100))
-	}
-	tabWriter.Flush() //nolint:errcheck
-
-	fmt.Fprintf(w, "- Status codes:\n")
-	for code, cnt := range r.statusCodes {
-		fmt.Fprintf(w, "    [%d]: %d\n", code, cnt)
-	}
-
-	// print the 10 most occurring errors (in case error values are not unique)
-	//
-	type kv struct {
-		err string
-		cnt int
-	}
-	var sortedErrors []kv
-	for err, cnt := range r.errors {
-		sortedErrors = append(sortedErrors, kv{err, cnt})
-	}
-	sort.Slice(sortedErrors, func(i, j int) bool {
-		return sortedErrors[i].cnt > sortedErrors[j].cnt
-	})
-	fmt.Fprintf(w, "- Errors (top 10):\n")
-	for i, se := range sortedErrors {
-		if i > 10 {
-			break
-		}
-		fmt.Fprintf(w, "    [%s]: %d\n", se.err, se.cnt)
-	}
-}
-
-type Bucket struct {
-	start int64
-	// the end value of the bucket
-	end int64
-	// how many entries are in the bucket
-	cnt int
-}

cmd/lotus-miner/sectors.go

@@ -2552,6 +2552,8 @@ var sectorsUnsealCmd = &cli.Command{
 			return xerrors.Errorf("could not parse sector number: %w", err)
 		}
 
+		fmt.Printf("Unsealing sector %d\n", sectorNum)
+
 		return minerAPI.SectorUnseal(ctx, abi.SectorNumber(sectorNum))
 	},
 }

cmd/lotus-shed/mpool.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/urfave/cli/v2"
 
@@ -43,10 +44,20 @@ var minerSelectMsgsCmd = &cli.Command{
 			return err
 		}
 
+		// Get the size of the mempool
+		pendingMsgs, err := api.MpoolPending(ctx, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+		mpoolSize := len(pendingMsgs)
+
+		// Measure the time taken by MpoolSelect
+		startTime := time.Now()
 		msgs, err := api.MpoolSelect(ctx, head.Key(), cctx.Float64("ticket-quality"))
 		if err != nil {
 			return err
 		}
+		duration := time.Since(startTime)
 
 		var totalGas int64
 		for i, f := range msgs {
@@ -64,6 +75,9 @@ var minerSelectMsgsCmd = &cli.Command{
 			totalGas += f.Message.GasLimit
 		}
 
+		// Log the duration, size of the mempool, selected messages and total gas limit of selected messages
+		fmt.Printf("Message selection took %s\n", duration)
+		fmt.Printf("Size of the mempool: %d\n", mpoolSize)
 		fmt.Println("selected messages: ", len(msgs))
 		fmt.Printf("total gas limit of selected messages: %d / %d (%0.2f%%)\n", totalGas, build.BlockGasLimit, 100*float64(totalGas)/float64(build.BlockGasLimit))
 		return nil

cmd/lotus-worker/main.go

@@ -627,7 +627,7 @@ var runCmd = &cli.Command{
 			Storage:    lr,
 		}
 
-		log.Info("Setting up control endpoint at " + address)
+		log.Info("Setting up control endpoint at " + newAddress)
 
 		timeout, err := time.ParseDuration(cctx.String("http-server-timeout"))
 		if err != nil {
@@ -652,13 +652,13 @@ var runCmd = &cli.Command{
 			log.Warn("Graceful shutdown successful")
 		}()
 
-		nl, err := net.Listen("tcp", address)
+		nl, err := net.Listen("tcp", newAddress)
 		if err != nil {
 			return err
 		}
 
 		{
-			a, err := net.ResolveTCPAddr("tcp", address)
+			a, err := net.ResolveTCPAddr("tcp", newAddress)
 			if err != nil {
 				return xerrors.Errorf("parsing address: %w", err)
 			}
@@ -739,7 +739,7 @@ var runCmd = &cli.Command{
 
 					select {
 					case <-readyCh:
-						if err := nodeApi.WorkerConnect(ctx, "http://"+address+"/rpc/v0"); err != nil {
+						if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil {
 							log.Errorf("Registering worker failed: %+v", err)
 							cancel()
 							return
@@ -801,15 +801,13 @@ func extractRoutableIP(timeout time.Duration) (string, error) {
 	}
 
 	minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6)
+	if minerIP == "" {
+		minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4)
+	}
 	minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP)
 
-	// Check if the IP is IPv6 and format the address appropriately
-	var addressToDial string
-	if ip := net.ParseIP(minerIP); ip.To4() == nil && ip.To16() != nil {
-		addressToDial = "[" + minerIP + "]:" + minerPort
-	} else {
-		addressToDial = minerIP + ":" + minerPort
-	}
+	// Format the address appropriately
+	addressToDial := net.JoinHostPort(minerIP, minerPort)
 
 	conn, err := net.DialTimeout("tcp", addressToDial, timeout)
 	if err != nil {

cmd/lotus/daemon.go

@@ -269,7 +269,37 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		if cctx.Bool("remove-existing-chain") {
+		chainfile := cctx.String("import-chain")
+		snapshot := cctx.String("import-snapshot")
+		willImportChain := false
+		if chainfile != "" || snapshot != "" {
+			if chainfile != "" && snapshot != "" {
+				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
+			}
+			willImportChain = true
+		}
+
+		willRemoveChain := cctx.Bool("remove-existing-chain")
+		if willImportChain && !willRemoveChain {
+			// Confirm with the user about the intention to remove chain data.
+			reader := bufio.NewReader(os.Stdin)
+			fmt.Print("Importing chain or snapshot will by default delete existing local chain data. Do you want to proceed and delete? (yes/no): ")
+			userInput, err := reader.ReadString('\n')
+			if err != nil {
+				return xerrors.Errorf("reading user input: %w", err)
+			}
+			userInput = strings.ToLower(strings.TrimSpace(userInput))
+
+			if userInput == "yes" {
+				willRemoveChain = true
+			} else if userInput == "no" {
+				willRemoveChain = false
+			} else {
+				return fmt.Errorf("invalid input. please answer with 'yes' or 'no'")
+			}
+		}
+
+		if willRemoveChain {
 			lr, err := repo.NewFS(cctx.String("repo"))
 			if err != nil {
 				return xerrors.Errorf("error opening fs repo: %w", err)
@@ -289,12 +319,7 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
-		chainfile := cctx.String("import-chain")
-		snapshot := cctx.String("import-snapshot")
-		if chainfile != "" || snapshot != "" {
-			if chainfile != "" && snapshot != "" {
-				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
-			}
+		if willImportChain {
 			var issnapshot bool
 			if chainfile == "" {
 				chainfile = snapshot

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.4-dev
+   1.23.5-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.4-dev
+   1.23.5-dev
 
 COMMANDS:
    run        Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.23.4-dev
+   1.23.5-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process

documentation/en/default-lotus-config.toml

@@ -195,7 +195,7 @@
 
   [Chainstore.Splitstore]
     # ColdStoreType specifies the type of the coldstore.
-    # It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
+    # It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state..
     #
     # type: string
     # env var: LOTUS_CHAINSTORE_SPLITSTORE_COLDSTORETYPE

go.mod

@@ -107,7 +107,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
-	github.com/libp2p/go-libp2p v0.30.0
+	github.com/libp2p/go-libp2p v0.31.0
 	github.com/libp2p/go-libp2p-consensus v0.0.1
 	github.com/libp2p/go-libp2p-gorpc v0.5.0
 	github.com/libp2p/go-libp2p-kad-dht v0.24.0
@@ -297,7 +297,7 @@ require (
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.37.6 // indirect
+	github.com/quic-go/quic-go v0.38.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect

go.sum

@@ -967,8 +967,8 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS
 github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw=
 github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o=
 github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0=
-github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U=
-github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg=
+github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg=
+github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
@@ -1456,8 +1456,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
 github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
 github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.37.6 h1:2IIUmQzT5YNxAiaPGjs++Z4hGOtIR0q79uS5qE9ccfY=
-github.com/quic-go/quic-go v0.37.6/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
+github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
+github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
 github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=

itests/api_test.go

@@ -86,7 +86,7 @@ func (ts *apiSuite) testID(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	require.Regexp(t, "^12", id.Pretty())
+	require.Regexp(t, "^12", id.String())
 }
 
 func (ts *apiSuite) testConnectTwo(t *testing.T) {

itests/eth_filter_test.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/xerrors"
 
@@ -543,6 +544,7 @@ func TestTxReceiptBloom(t *testing.T) {
 		kit.MockProofs(),
 		kit.ThroughRPC())
 	ens.InterconnectAll().BeginMining(blockTime)
+	logging.SetLogLevel("fullnode", "DEBUG")
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
@@ -577,6 +579,10 @@ func TestTxReceiptBloom(t *testing.T) {
 		}
 	}
 
+	// Deflake plan: (Flake: 5 bits instead of 6)
+	//   Debug + search logs for "LogsBloom"
+	//   compare to passing case.
+	//
 	// 3 bits from the topic, 3 bits from the address
 	require.Equal(t, 6, bitsSet)
 }

lib/consensus/raft/consensus.go

@@ -342,7 +342,7 @@ func (cc *Consensus) RedirectToLeader(method string, arg interface{}, ret interf
 			return false, nil
 		}
 
-		logger.Debugf("redirecting %s to leader: %s", method, leader.Pretty())
+		logger.Debugf("redirecting %s to leader: %s", method, leader)
 		finalErr = cc.RpcClient.CallContext(
 			ctx,
 			leader,
@@ -394,7 +394,7 @@ func (cc *Consensus) Commit(ctx context.Context, op *ConsensusOp) error {
 func (cc *Consensus) AddPeer(ctx context.Context, pid peer.ID) error {
 	var finalErr error
 	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: AddPeer %s", i, pid.Pretty())
+		logger.Debugf("attempt #%d: AddPeer %s", i, pid)
 		if finalErr != nil {
 			logger.Errorf("retrying to add peer. Attempt #%d failed: %s", i, finalErr)
 		}
@@ -408,7 +408,7 @@ func (cc *Consensus) AddPeer(ctx context.Context, pid peer.ID) error {
 			time.Sleep(cc.config.CommitRetryDelay)
 			continue
 		}
-		logger.Infof("peer added to Raft: %s", pid.Pretty())
+		logger.Infof("peer added to Raft: %s", pid)
 		break
 	}
 	return finalErr
@@ -419,7 +419,7 @@ func (cc *Consensus) AddPeer(ctx context.Context, pid peer.ID) error {
 func (cc *Consensus) RmPeer(ctx context.Context, pid peer.ID) error {
 	var finalErr error
 	for i := 0; i <= cc.config.CommitRetries; i++ {
-		logger.Debugf("attempt #%d: RmPeer %s", i, pid.Pretty())
+		logger.Debugf("attempt #%d: RmPeer %s", i, pid)
 		if finalErr != nil {
 			logger.Errorf("retrying to remove peer. Attempt #%d failed: %s", i, finalErr)
 		}
@@ -433,7 +433,7 @@ func (cc *Consensus) RmPeer(ctx context.Context, pid peer.ID) error {
 			time.Sleep(cc.config.CommitRetryDelay)
 			continue
 		}
-		logger.Infof("peer removed from Raft: %s", pid.Pretty())
+		logger.Infof("peer removed from Raft: %s", pid)
 		break
 	}
 	return finalErr

lib/consensus/raft/raft.go

@@ -561,33 +561,3 @@ func find(s []string, elem string) bool {
 	}
 	return false
 }
-
-func (rw *raftWrapper) observePeers() {
-	obsCh := make(chan hraft.Observation, 1)
-	defer close(obsCh)
-
-	observer := hraft.NewObserver(obsCh, true, func(o *hraft.Observation) bool {
-		po, ok := o.Data.(hraft.PeerObservation)
-		return ok && po.Removed
-	})
-
-	rw.raft.RegisterObserver(observer)
-	defer rw.raft.DeregisterObserver(observer)
-
-	for {
-		select {
-		case obs := <-obsCh:
-			pObs := obs.Data.(hraft.PeerObservation)
-			logger.Info("raft peer departed. Removing from peerstore: ", pObs.Peer.ID)
-			pID, err := peer.Decode(string(pObs.Peer.ID))
-			if err != nil {
-				logger.Error(err)
-				continue
-			}
-			rw.host.Peerstore().ClearAddrs(pID)
-		case <-rw.ctx.Done():
-			logger.Debug("stopped observing raft peers")
-			return
-		}
-	}
-}

lib/shardedmutex/shardedmutex.go

@@ -11,8 +11,8 @@ const cacheline = 64
 // name     old time/op  new time/op  delta
 // Locks-8  74.6ns ± 7%  12.3ns ± 2%  -83.54%  (p=0.000 n=20+18)
 type paddedMutex struct {
-	mt  sync.Mutex
-	pad [cacheline - 8]uint8
+	mt sync.Mutex
+	_  [cacheline - 8]uint8
 }
 
 type ShardedMutex struct {

metrics/README.md

@@ -0,0 +1,144 @@
+# Setting Up Prometheus and Grafana
+
+Lotus supports exporting a wide range of metrics, enabling users to gain insights into its behavior and effectively analyze performance issues. These metrics can be conveniently utilized with aggregation and visualization tools for in-depth analysis. In this document, we show how you can set up Prometheus and Grafana for monitoring and visualizing these metrics:
+
+- **Prometheus**: Prometheus is an open-source monitoring and alerting toolkit designed for collecting and storing time-series data from various systems and applications. It provides a robust querying language (PromQL) and a web-based interface for analyzing and visualizing metrics.
+
+- **Grafana**: Grafana is an open-source platform for creating, sharing, and visualizing interactive dashboards and graphs. It integrates with various data sources, including Prometheus, to help users create meaningful visual representations of their data and set up alerting based on specific conditions.
+
+## Prerequisites
+
+- You have a Linux or Mac based system.
+- You have root access to install software
+- You have lotus node already running
+
+**Note:** These instructions have been tested on Ubuntu 23.04 and on Mac M1.
+
+## Install and start Prometheus
+
+### On Ubuntu:
+
+```
+# install prometheus
+sudo apt-get install prometheus
+
+# copy the prometheus.yml config to the correct directory
+sudo cp metrics/prometheus.yml /etc/prometheus/prometheus.yml
+
+# start prometheus
+sudo systemctl start prometheus
+
+# enable prometheus on boot (optional)
+sudo systemctl enable prometheus
+```
+
+### On Mac:
+
+```
+# install prometheus
+brew install prometheus
+
+# start prometheus
+prometheus --config.file=lotus/metrics/prometheus.yml
+```
+
+## Install and start Grafana
+
+### On Ubuntu:
+
+```
+# download the Grafana GPG key in our keyring
+wget -q -O - https://packages.grafana.com/gpg.key | gpg --dearmor | sudo tee /usr/share/keyrings/grafana.gpg > /dev/null
+
+# add the Grafana repository to our APT sources
+echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
+
+# update our APT cache
+sudo apt-get update
+
+# now we can install grafana
+sudo apt-get install grafana
+
+# start grafana
+sudo systemctl start grafana-server
+
+# start grafana on boot (optional)
+sudo systemctl enable grafana-server
+```
+
+### On Mac:
+
+```
+brew install grafana
+brew services start grafana
+```
+
+You should now have Prometheus and Grafana running on your machine where Prometheus is already collecting metrics from your Lotus node (if its running) and saving it to a database.
+
+You can confirm everything is setup correctly by visiting:
+- Prometheus (http://localhost:9090): You can open the metric explorer and view any of the aggregated metrics scraped from Lotus
+- Grafana (http://localhost:3000): Default username/password is admin/admin, remember to change it after login.
+
+## Add Prometheus as datasource in Grafana
+
+1. Log in to Grafana using the web interface.
+2. Navigate to "Home" > "Connections" > "Data Sources."
+3. Click "Add data source."
+4. Choose "Prometheus."
+5. In the "HTTP" section, set the URL to http://localhost:9090.
+6. Click "Save & Test" to verify the connection.
+
+## Import one of the existing dashboards in lotus/metrics/grafana
+
+1. Log in to Grafana using the web interface.
+2. Navigate to "Home" > "Dashboards" > Click the drop down menu in the "New" button and select "Import"
+3. Paste any of the existing dashboards in lotus/metrics/grafana into the "Import via panel json" panel.
+4. Click "Load"
+5. Select the Prometheus datasource you created earlier
+6. Click "Import"
+
+# Collect system metrics using node_exporter
+
+Although Lotus includes many useful metrics it does not include system metrics, such as information about cpu, memory, disk, network, etc. If you are investigating an issue and have Lotus metrics available, its often very useful to correlate certain events or behaviour with general system metrics.
+
+## Install node_exporter
+If you have followed this guide so far and have Prometheus and Grafana already running, you can run the following commands to also aggregate the system metrics:
+
+Ubuntu:
+
+```
+
+# download the newest release by https://github.com/prometheus/node_exporter/releases (it was 1.6.1 as of writing this doc)
+wget https://github.com/prometheus/node_exporter/releases/download/v1.6.1/node_exporter-1.6.1.linux-amd64.tar.gz
+
+# extract the release (it contains a single binary plus some docs)
+tar -xf node_exporter-1.6.1.linux-amd64.tar.gz
+
+# move it to /usr/local/bin
+sudo mv node_exporter-1.6.1.linux-amd64/node_exporter /usr/local/bin
+
+# run node_exporter
+node_exporter
+```
+
+Mac:
+
+```
+# install node_exporter
+brew install node_exporter
+
+# run node_exporter
+node_exporter
+```
+
+## Import system dashboard
+
+Since our `prometheus.yml` config already has configuration for node_exporter, we can go straight away and import a Grafana dashboard for viewing:
+
+1. Download the most recent dashboard from https://grafana.com/grafana/dashboards/1860-node-exporter-full/
+2. Log in to Grafana (http://localhost:3000) using the web interface.
+3. Navigate to "Home" > "Dashboards" > Click the drop down menu in the "New" button and select "Import"
+4. Paste any of the existing dashboards in lotus/metrics/grafana into the "Import via panel json" panel.
+5. Click "Load"
+6. Select the Prometheus datasource you created earlier
+7. Click "Import"

metrics/grafana/MessageExecution.json

@@ -0,0 +1,241 @@
+{
+    "__inputs": [
+        {
+            "name": "DS_PROMETHEUS",
+            "label": "Prometheus",
+            "description": "",
+            "type": "datasource",
+            "pluginId": "prometheus",
+            "pluginName": "Prometheus"
+        }
+    ],
+    "__elements": {},
+    "__requires": [
+        {
+            "type": "grafana",
+            "id": "grafana",
+            "name": "Grafana",
+            "version": "10.1.1"
+        },
+        {
+            "type": "datasource",
+            "id": "prometheus",
+            "name": "Prometheus",
+            "version": "1.0.0"
+        },
+        {
+            "type": "panel",
+            "id": "timeseries",
+            "name": "Time series",
+            "version": ""
+        }
+    ],
+    "annotations": {
+        "list": [
+            {
+                "builtIn": 1,
+                "datasource": {
+                    "type": "grafana",
+                    "uid": "-- Grafana --"
+                },
+                "enable": true,
+                "hide": true,
+                "iconColor": "rgba(0, 211, 255, 1)",
+                "name": "Annotations & Alerts",
+                "type": "dashboard"
+            }
+        ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": null,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+        {
+            "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+            },
+            "description": "Understand where time is spent in ApplyBlocks which is executed as part of ExecuteTipSet, its metric include:\n\n- applyblocks_total_ms (total): The total time spent in Applyblocks\n- applyblocks_cron (cron): Time spent in cron\n- applyblocks_early (early): Time spent in early apply-blocks (null cron, upgrades)\n- applyblocks_flush (flush): Time spent flushing vm state\n- applyblocks_messages (apply messages): Time spent applying block messages\n",
+            "fieldConfig": {
+                "defaults": {
+                    "color": {
+                        "mode": "palette-classic"
+                    },
+                    "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "Time in MS",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                            "legend": false,
+                            "tooltip": false,
+                            "viz": false
+                        },
+                        "insertNulls": false,
+                        "lineInterpolation": "smooth",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                            "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": 60000,
+                        "stacking": {
+                            "group": "A",
+                            "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                            "mode": "off"
+                        }
+                    },
+                    "mappings": [],
+                    "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                            {
+                                "color": "green",
+                                "value": null
+                            },
+                            {
+                                "color": "red",
+                                "value": 80
+                            }
+                        ]
+                    }
+                },
+                "overrides": []
+            },
+            "gridPos": {
+                "h": 10,
+                "w": 12,
+                "x": 0,
+                "y": 0
+            },
+            "id": 1,
+            "options": {
+                "legend": {
+                    "calcs": [],
+                    "displayMode": "list",
+                    "placement": "bottom",
+                    "showLegend": true
+                },
+                "tooltip": {
+                    "mode": "single",
+                    "sort": "none"
+                }
+            },
+            "targets": [
+                {
+                    "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                    },
+                    "disableTextWrap": false,
+                    "editorMode": "builder",
+                    "expr": "histogram_quantile(0.99, sum by(le) (rate(lotus_vm_applyblocks_total_ms_bucket[$__rate_interval])))",
+                    "fullMetaSearch": false,
+                    "includeNullMetadata": false,
+                    "instant": false,
+                    "legendFormat": "Total",
+                    "range": true,
+                    "refId": "A",
+                    "useBackend": false
+                },
+                {
+                    "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                    },
+                    "disableTextWrap": false,
+                    "editorMode": "builder",
+                    "expr": "histogram_quantile(0.99, sum by(le) (rate(lotus_vm_applyblocks_cron_bucket[$__rate_interval])))",
+                    "fullMetaSearch": false,
+                    "hide": false,
+                    "includeNullMetadata": false,
+                    "instant": false,
+                    "legendFormat": "Cron",
+                    "range": true,
+                    "refId": "B",
+                    "useBackend": false
+                },
+                {
+                    "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                    },
+                    "disableTextWrap": false,
+                    "editorMode": "builder",
+                    "expr": "histogram_quantile(0.99, sum by(le) (rate(lotus_vm_applyblocks_early_bucket[$__rate_interval])))",
+                    "fullMetaSearch": false,
+                    "hide": false,
+                    "includeNullMetadata": false,
+                    "instant": false,
+                    "legendFormat": "Early",
+                    "range": true,
+                    "refId": "C",
+                    "useBackend": false
+                },
+                {
+                    "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                    },
+                    "disableTextWrap": false,
+                    "editorMode": "builder",
+                    "expr": "histogram_quantile(0.99, sum by(le) (rate(lotus_vm_applyblocks_flush_bucket[$__rate_interval])))",
+                    "fullMetaSearch": false,
+                    "hide": false,
+                    "includeNullMetadata": false,
+                    "instant": false,
+                    "legendFormat": "Flush",
+                    "range": true,
+                    "refId": "D",
+                    "useBackend": false
+                },
+                {
+                    "datasource": {
+                        "type": "prometheus",
+                        "uid": "${DS_PROMETHEUS}"
+                    },
+                    "disableTextWrap": false,
+                    "editorMode": "builder",
+                    "expr": "histogram_quantile(0.99, sum by(le) (rate(lotus_vm_applyblocks_messages_bucket[$__rate_interval])))",
+                    "fullMetaSearch": false,
+                    "hide": false,
+                    "includeNullMetadata": false,
+                    "instant": false,
+                    "legendFormat": "Apply messages",
+                    "range": true,
+                    "refId": "E",
+                    "useBackend": false
+                }
+            ],
+            "title": "ApplyBlocks (ms)",
+            "type": "timeseries"
+        }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+        "list": []
+    },
+    "time": {
+        "from": "now-5m",
+        "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Lotus Message Execution",
+    "uid": "a7bacd0e-f7a1-418f-98e5-3469c5e0b6ea",
+    "version": 5,
+    "weekStart": ""
+}

metrics/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 1m
+
+scrape_configs:
+  - job_name: lotus
+    scrape_interval: 10s
+    metrics_path: '/debug/metrics'
+    static_configs:
+      - targets: ['localhost:1234']
+
+  - job_name: node_exporter
+    scrape_interval: 15s
+    static_configs:
+      - targets: ['localhost:9100']

miner/miner.go

@@ -562,7 +562,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (minedBlock *type
 		return nil, err
 	}
 
-	tEquivocateWait := build.Clock.Now()
+	tPending := build.Clock.Now()
 
 	// This next block exists to "catch" equivocating miners,
 	// who submit 2 blocks at the same height at different times in order to split the network.
@@ -576,6 +576,8 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (minedBlock *type
 		return nil, err
 	}
 
+	tEquivocateWait := build.Clock.Now()
+
 	// If the base has changed, we take the _intersection_ of our old base and new base,
 	// thus ejecting blocks from any equivocating miners, without taking any new blocks.
 	if newBase.TipSet.Height() == base.TipSet.Height() && !newBase.TipSet.Equals(base.TipSet) {
@@ -621,7 +623,7 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (minedBlock *type
 		}
 	}
 
-	tPending := build.Clock.Now()
+	tIntersectAndRefresh := build.Clock.Now()
 
 	// TODO: winning post proof
 	minedBlock, err = m.createBlock(base, m.address, ticket, winner, bvals, postProof, msgs)
@@ -637,15 +639,16 @@ func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (minedBlock *type
 		parentMiners[i] = header.Miner
 	}
 	log.Infow("mined new block", "cid", minedBlock.Cid(), "height", int64(minedBlock.Header.Height), "miner", minedBlock.Header.Miner, "parents", parentMiners, "parentTipset", base.TipSet.Key().String(), "took", dur)
-	if dur > time.Second*time.Duration(build.BlockDelaySecs) {
-		log.Warnw("CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up",
+	if dur > time.Second*time.Duration(build.BlockDelaySecs) || time.Now().Compare(time.Unix(int64(minedBlock.Header.Timestamp), 0)) >= 0 {
+		log.Warnw("CAUTION: block production took us past the block time. Your computer may not be fast enough to keep up",
 			"tPowercheck ", tPowercheck.Sub(tStart),
 			"tTicket ", tTicket.Sub(tPowercheck),
 			"tSeed ", tSeed.Sub(tTicket),
 			"tProof ", tProof.Sub(tSeed),
-			"tEquivocateWait ", tEquivocateWait.Sub(tProof),
-			"tPending ", tPending.Sub(tEquivocateWait),
-			"tCreateBlock ", tCreateBlock.Sub(tPending))
+			"tPending ", tPending.Sub(tProof),
+			"tEquivocateWait ", tEquivocateWait.Sub(tPending),
+			"tIntersectAndRefresh ", tIntersectAndRefresh.Sub(tEquivocateWait),
+			"tCreateBlock ", tCreateBlock.Sub(tIntersectAndRefresh))
 	}
 
 	return minedBlock, nil

node/config/doc_gen.go

@@ -1302,7 +1302,7 @@ Submitting a smaller number of prove commits per epoch would reduce the possibil
 			Type: "string",
 
 			Comment: `ColdStoreType specifies the type of the coldstore.
-It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.`,
+It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state..`,
 		},
 		{
 			Name: "HotStoreType",

node/config/types.go

@@ -585,7 +585,7 @@ type Chainstore struct {
 
 type Splitstore struct {
 	// ColdStoreType specifies the type of the coldstore.
-	// It can be "messages" (default) to store only messages, "universal" to store all chain state or "discard" for discarding cold blocks.
+	// It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state..
 	ColdStoreType string
 	// HotStoreType specifies the type of the hotstore.
 	// Only currently supported value is "badger".

node/impl/full/eth_utils.go

@@ -665,6 +665,7 @@ func newEthTxReceipt(ctx context.Context, tx ethtypes.EthTx, lookup *api.MsgLook
 				continue
 			}
 			for _, topic := range topics {
+				log.Debug("LogsBloom set for ", topic)
 				ethtypes.EthBloomSet(receipt.LogsBloom, topic[:])
 			}
 			l.Data = data

node/impl/net/rcmgr.go

@@ -36,7 +36,7 @@ func (a *NetAPI) NetStat(ctx context.Context, scope string) (result api.NetStat,
 		if len(stat.Peers) > 0 {
 			result.Peers = make(map[string]network.ScopeStat, len(stat.Peers))
 			for p, stat := range stat.Peers {
-				result.Peers[p.Pretty()] = stat
+				result.Peers[p.String()] = stat
 			}
 		}
 

node/modules/actorevent.go

@@ -126,8 +126,6 @@ func EthEventAPI(cfg config.FevmConfig) func(helpers.MetricsCtx, repo.LockedRepo
 			MaxFilterResults: cfg.Events.MaxFilterResults,
 		}
 
-		const ChainHeadConfidence = 1
-
 		lc.Append(fx.Hook{
 			OnStart: func(context.Context) error {
 				ev, err := events.NewEvents(ctx, &evapi)

node/modules/lp2p/host.go

@@ -41,7 +41,7 @@ func Peerstore() (peerstore.Peerstore, error) {
 func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (RawHost, error) {
 	pkey := params.Peerstore.PrivKey(params.ID)
 	if pkey == nil {
-		return nil, fmt.Errorf("missing private key for node ID: %s", params.ID.Pretty())
+		return nil, fmt.Errorf("missing private key for node ID: %s", params.ID)
 	}
 
 	opts := []libp2p.Option{

node/modules/services.go

@@ -11,8 +11,8 @@ import (
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/event"
 	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/libp2p/go-libp2p/p2p/host/eventbus"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
@@ -66,18 +66,22 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.
 	ctx := helpers.LifecycleCtx(mctx, lc)
 
 	go func() {
+		// We want to get information on connected peers, we don't want to trigger new connections.
+		ctx := network.WithNoDial(ctx, "filecoin hello")
 		for evt := range sub.Out() {
 			pic := evt.(event.EvtPeerIdentificationCompleted)
+			// We just finished identifying the peer, that means we should know what
+			// protocols it speaks. Check if it speeks the Filecoin hello protocol
+			// before continuing.
+			if p, _ := h.Peerstore().FirstSupportedProtocol(pic.Peer, hello.ProtocolID); p != hello.ProtocolID {
+				continue
+			}
+
 			go func() {
 				if err := svc.SayHello(ctx, pic.Peer); err != nil {
 					protos, _ := h.Peerstore().GetProtocols(pic.Peer)
 					agent, _ := h.Peerstore().Get(pic.Peer, "AgentVersion")
-					if protosContains(protos, hello.ProtocolID) {
-						log.Warnw("failed to say hello", "error", err, "peer", pic.Peer, "supported", protos, "agent", agent)
-					} else {
-						log.Debugw("failed to say hello", "error", err, "peer", pic.Peer, "supported", protos, "agent", agent)
-					}
-					return
+					log.Warnw("failed to say hello", "error", err, "peer", pic.Peer, "supported", protos, "agent", agent)
 				}
 			}()
 		}
@@ -85,15 +89,6 @@ func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.
 	return nil
 }
 
-func protosContains(protos []protocol.ID, search protocol.ID) bool {
-	for _, p := range protos {
-		if p == search {
-			return true
-		}
-	}
-	return false
-}
-
 func RunPeerMgr(mctx helpers.MetricsCtx, lc fx.Lifecycle, pmgr *peermgr.PeerMgr) {
 	go pmgr.Run(helpers.LifecycleCtx(mctx, lc))
 }

paychmgr/paych_test.go

@@ -805,26 +805,6 @@ func createTestVoucher(t *testing.T, ch address.Address, voucherLane uint64, non
 	return sv
 }
 
-func createTestVoucherWithExtra(t *testing.T, ch address.Address, voucherLane uint64, nonce uint64, voucherAmount big.Int, key []byte) *paychtypes.SignedVoucher { //nolint:deadcode
-	sv := &paychtypes.SignedVoucher{
-		ChannelAddr: ch,
-		Lane:        voucherLane,
-		Nonce:       nonce,
-		Amount:      voucherAmount,
-		Extra: &paychtypes.ModVerifyParams{
-			Actor: tutils.NewActorAddr(t, "act"),
-		},
-	}
-
-	signingBytes, err := sv.SigningBytes()
-	require.NoError(t, err)
-	sig, err := sigs.Sign(crypto.SigTypeSecp256k1, key, signingBytes)
-	require.NoError(t, err)
-	sv.Signature = sig
-
-	return sv
-}
-
 type mockBestSpendableAPI struct {
 	mgr *Manager
 }

storage/sealer/ffiwrapper/sealer_test.go

@@ -425,6 +425,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestSealAndVerify3(t *testing.T) {
+	t.Skip("i flake on CI, re-enable me when you have a fix pls")
 	if testing.Short() {
 		t.Skip("skipping test in short mode")
 	}

storage/sealer/manager.go

@@ -802,7 +802,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
 
 	{
 		unsealedStores, ferr := m.index.StorageFindSector(ctx, sector.ID, storiface.FTUnsealed, 0, false)
-		if err != nil {
+		if ferr != nil {
 			err = multierr.Append(err, xerrors.Errorf("find unsealed sector before move: %w", ferr))
 		} else if len(unsealedStores) > 0 {
 			// if we found unsealed files, AND have been asked to keep at least one piece, move unsealed

storage/sealer/proofpaths/cachefiles_test.go

@@ -10,6 +10,10 @@ import (
 
 func TestSDRLayersDefined(t *testing.T) {
 	for proof := range abi.SealProofInfos {
+		// TODO: Drop after feat/nv21 is merged in (that is, when SynthPoRep changes land)
+		if proof >= abi.RegisteredSealProof_StackedDrg2KiBV1_1_Feat_SyntheticPoRep && proof <= abi.RegisteredSealProof_StackedDrg64GiBV1_1_Feat_SyntheticPoRep {
+			continue
+		}
 		_, err := SDRLayers(proof)
 		require.NoError(t, err)
 	}

storage/wdpost/wdpost_run.go

@@ -32,6 +32,8 @@ import (
 )
 
 // recordPoStFailure records a failure in the journal.
+//
+//nolint:unused
 func (s *WindowPoStScheduler) recordPoStFailure(err error, ts *types.TipSet, deadline *dline.Info) {
 	s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
 		c := evtCommon{Error: err}

storage/wdpost/wdpost_run_faults.go

@@ -195,106 +195,6 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6
 	return batchedRecoveryDecls, msgs, nil
 }
 
-// declareFaults identifies the sectors on the specified proving deadline that
-// are faulty, and reports the faults on chain via the `DeclareFaults` message
-// to our miner actor.
-//
-// NOTE: THIS CODE ISN'T INVOKED AFTER THE IGNITION UPGRADE
-// This is always invoked ahead of time, before the deadline for the evaluated
-// sectors arrives. That way, faults are declared before a penalty is accrued.
-//
-// If a declaration is made, it awaits for build.MessageConfidence confirmations
-// on chain before returning.
-//
-// TODO: the waiting should happen in the background. Right now this
-//
-//	is blocking/delaying the actual generation and submission of WindowPoSts in
-//	this deadline!
-func (s *WindowPoStScheduler) declareFaults(ctx context.Context, dlIdx uint64, partitions []api.Partition, tsk types.TipSetKey) ([]miner.FaultDeclaration, *types.SignedMessage, error) {
-	ctx, span := trace.StartSpan(ctx, "storage.declareFaults")
-	defer span.End()
-
-	bad := uint64(0)
-	params := &miner.DeclareFaultsParams{
-		Faults: []miner.FaultDeclaration{},
-	}
-
-	for partIdx, partition := range partitions {
-		nonFaulty, err := bitfield.SubtractBitField(partition.LiveSectors, partition.FaultySectors)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("determining non faulty sectors: %w", err)
-		}
-
-		good, err := s.checkSectors(ctx, nonFaulty, tsk)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("checking sectors: %w", err)
-		}
-
-		newFaulty, err := bitfield.SubtractBitField(nonFaulty, good)
-		if err != nil {
-			return nil, nil, xerrors.Errorf("calculating faulty sector set: %w", err)
-		}
-
-		c, err := newFaulty.Count()
-		if err != nil {
-			return nil, nil, xerrors.Errorf("counting faulty sectors: %w", err)
-		}
-
-		if c == 0 {
-			continue
-		}
-
-		bad += c
-
-		params.Faults = append(params.Faults, miner.FaultDeclaration{
-			Deadline:  dlIdx,
-			Partition: uint64(partIdx),
-			Sectors:   newFaulty,
-		})
-	}
-
-	faults := params.Faults
-	if len(faults) == 0 {
-		return faults, nil, nil
-	}
-
-	log.Errorw("DETECTED FAULTY SECTORS, declaring faults", "count", bad)
-
-	enc, aerr := actors.SerializeParams(params)
-	if aerr != nil {
-		return faults, nil, xerrors.Errorf("could not serialize declare faults parameters: %w", aerr)
-	}
-
-	msg := &types.Message{
-		To:     s.actor,
-		Method: builtin.MethodsMiner.DeclareFaults,
-		Params: enc,
-		Value:  types.NewInt(0), // TODO: Is there a fee?
-	}
-	spec := &api.MessageSendSpec{MaxFee: abi.TokenAmount(s.feeCfg.MaxWindowPoStGasFee)}
-	if err := s.prepareMessage(ctx, msg, spec); err != nil {
-		return faults, nil, err
-	}
-
-	sm, err := s.api.MpoolPushMessage(ctx, msg, spec)
-	if err != nil {
-		return faults, sm, xerrors.Errorf("pushing message to mpool: %w", err)
-	}
-
-	log.Warnw("declare faults Message CID", "cid", sm.Cid())
-
-	rec, err := s.api.StateWaitMsg(context.TODO(), sm.Cid(), build.MessageConfidence, api.LookbackNoLimit, true)
-	if err != nil {
-		return faults, sm, xerrors.Errorf("declare faults wait error: %w", err)
-	}
-
-	if rec.Receipt.ExitCode != 0 {
-		return faults, sm, xerrors.Errorf("declare faults wait non-0 exit code: %d", rec.Receipt.ExitCode)
-	}
-
-	return faults, sm, nil
-}
-
 func (s *WindowPoStScheduler) asyncFaultRecover(di dline.Info, ts *types.TipSet) {
 	go func() {
 		// check faults / recoveries for the *next* deadline. It's already too

storage/wdpost/wdpost_sched.go

@@ -225,6 +225,8 @@ func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.T
 }
 
 // onAbort is called when generating proofs or submitting proofs is aborted
+//
+//nolint:unused
 func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) {
 	s.journal.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
 		c := evtCommon{}