From 7710a57fa3c263bb9c9e336edcd8bb40c2fb372b Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Wed, 5 Apr 2023 15:25:57 -0400
Subject: [PATCH 01/46] fix: include extra messages in ComputeState InvocResult
 output

---
 chain/stmgr/utils.go | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go
index c93267d50..5e3bbd278 100644
--- a/chain/stmgr/utils.go
+++ b/chain/stmgr/utils.go
@@ -72,7 +72,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 
 	base, trace, err := sm.ExecutionTrace(ctx, ts)
 	if err != nil {
-		return cid.Undef, nil, err
+		return cid.Undef, nil, xerrors.Errorf("failed to compute base state: %w", err)
 	}
 
 	for i := ts.Height(); i < height; i++ {
@@ -116,6 +116,21 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		if ret.ExitCode != 0 {
 			log.Infof("compute state apply message %d failed (exit: %d): %s", i, ret.ExitCode, ret.ActorErr)
 		}
+
+		ir := &api.InvocResult{
+			MsgCid:         msg.Cid(),
+			Msg:            msg,
+			MsgRct:         &ret.MessageReceipt,
+			ExecutionTrace: ret.ExecutionTrace,
+			Duration:       ret.Duration,
+		}
+		if ret.ActorErr != nil {
+			ir.Error = ret.ActorErr.Error()
+		}
+		if ret.GasCosts != nil {
+			ir.GasCost = MakeMsgGasCost(msg, ret)
+		}
+		trace = append(trace, ir)
 	}
 
 	root, err := vmi.Flush(ctx)

From ae84f335ccfa1e780ec1b4e5a192bb526f74ffd0 Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Tue, 11 Apr 2023 10:26:43 -0400
Subject: [PATCH 02/46] feat: pubsub: treat ErrGasFeeCapTooLow as ignore, not
 reject

---
 chain/sub/incoming.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go
index 87598bd2e..5f15c4ad5 100644
--- a/chain/sub/incoming.go
+++ b/chain/sub/incoming.go
@@ -377,6 +377,8 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs
 			fallthrough
 		case xerrors.Is(err, messagepool.ErrNonceGap):
 			fallthrough
+		case xerrors.Is(err, messagepool.ErrGasFeeCapTooLow):
+			fallthrough
 		case xerrors.Is(err, messagepool.ErrNonceTooLow):
 			return pubsub.ValidationIgnore
 		default:

From 8fcf59facca1b79e435722866074a41e44a2444c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Sat, 22 Apr 2023 12:47:11 +0200
Subject: [PATCH 03/46] itests: Test PoSt V1_1 on workers

---
 itests/worker_test.go | 230 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 230 insertions(+)

diff --git a/itests/worker_test.go b/itests/worker_test.go
index 4e845afe8..87bb0047d 100644
--- a/itests/worker_test.go
+++ b/itests/worker_test.go
@@ -1,6 +1,7 @@
 package itests
 
 import (
+	"bytes"
 	"context"
 	"strings"
 	"sync/atomic"
@@ -13,13 +14,17 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner"
+	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/itests/kit"
 	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/impl"
+	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/storage/paths"
 	sealing "github.com/filecoin-project/lotus/storage/pipeline"
@@ -500,3 +505,228 @@ func TestWorkerName(t *testing.T) {
 
 	require.True(t, found)
 }
+
+// Tests that V1_1 proofs on post workers with faults
+func TestWindowPostV1P1NV20WorkerFault(t *testing.T) {
+	kit.QuietMiningLogs()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	blocktime := 2 * time.Millisecond
+
+	sectors := 2 * 48 * 2
+	var badsector uint64 = 100000
+
+	client, miner, _, ens := kit.EnsembleWorker(t,
+		kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
+		kit.GenesisNetworkVersion(network.Version20),
+		kit.ConstructorOpts(
+			node.Override(new(config.ProvingConfig), func() config.ProvingConfig {
+				c := config.DefaultStorageMiner()
+				c.Proving.DisableBuiltinWindowPoSt = true
+				return c.Proving
+			}),
+			node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
+				config.DefaultStorageMiner().Fees,
+				config.ProvingConfig{
+					DisableBuiltinWindowPoSt:  true,
+					DisableBuiltinWinningPoSt: false,
+					DisableWDPoStPreChecks:    false,
+				},
+			)),
+			node.Override(new(paths.Store), func(store *paths.Remote) paths.Store {
+				return &badWorkerStorage{
+					Store:       store,
+					badsector:   &badsector,
+					notBadCount: 1,
+				}
+			})),
+		kit.ThroughRPC(),
+		kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}),
+		kit.WithWorkerStorage(func(store paths.Store) paths.Store {
+			return &badWorkerStorage{
+				Store:     store,
+				badsector: &badsector,
+			}
+		}))
+
+	bm := ens.InterconnectAll().BeginMining(blocktime)[0]
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+
+	// wait for some crons(?)
+	require.Eventually(t, func() bool {
+		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+		require.NoError(t, err)
+
+		parts, err := client.StateMinerPartitions(ctx, maddr, di.Index, types.EmptyTSK)
+		require.NoError(t, err)
+
+		return len(parts) > 1
+	}, 30*time.Second, 100*time.Millisecond)
+
+	// Wait until just before a deadline opens
+	{
+		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+		require.NoError(t, err)
+
+		di = di.NextNotElapsed()
+
+		t.Log("Running one proving period")
+		waitUntil := di.Open + di.WPoStChallengeWindow - di.WPoStChallengeLookback - 1
+		client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
+
+		t.Log("Waiting for post message")
+		bm.Stop()
+	}
+
+	// Remove one sector in the next deadline (so it's skipped)
+	{
+		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
+		require.NoError(t, err)
+
+		parts, err := client.StateMinerPartitions(ctx, maddr, di.Index+1, types.EmptyTSK)
+		require.NoError(t, err)
+		require.Greater(t, len(parts), 0)
+
+		secs := parts[0].AllSectors
+		n, err := secs.Count()
+		require.NoError(t, err)
+		require.Equal(t, uint64(2), n)
+
+		// Drop the sector in second partition
+		sid, err := secs.First()
+		require.NoError(t, err)
+
+		t.Logf("Drop sector %d; dl %d part %d", sid, di.Index+1, 0)
+
+		atomic.StoreUint64(&badsector, sid)
+		require.NoError(t, err)
+	}
+
+	bm.MineBlocksMustPost(ctx, 2*time.Millisecond)
+
+	mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
+	require.NoError(t, err)
+	en := wact.Nonce
+
+	// wait for a new message to be sent from worker address, it will be a PoSt
+
+waitForProof:
+	for {
+		//stm: @CHAIN_STATE_GET_ACTOR_001
+		wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
+		require.NoError(t, err)
+		if wact.Nonce > en {
+			break waitForProof
+		}
+
+		build.Clock.Sleep(blocktime)
+	}
+
+	slm, err := client.StateListMessages(ctx, &api.MessageMatch{To: maddr}, types.EmptyTSK, 0)
+	require.NoError(t, err)
+
+	pmr, err := client.StateSearchMsg(ctx, types.EmptyTSK, slm[0], -1, false)
+	require.NoError(t, err)
+
+	nv, err := client.StateNetworkVersion(ctx, pmr.TipSet)
+	require.NoError(t, err)
+	require.Equal(t, network.Version20, nv)
+
+	require.True(t, pmr.Receipt.ExitCode.IsSuccess())
+
+	slmsg, err := client.ChainGetMessage(ctx, slm[0])
+	require.NoError(t, err)
+
+	var params miner11.SubmitWindowedPoStParams
+	require.NoError(t, params.UnmarshalCBOR(bytes.NewBuffer(slmsg.Params)))
+	require.Equal(t, abi.RegisteredPoStProof_StackedDrgWindow2KiBV1_1, params.Proofs[0].PoStProof)
+
+	require.Len(t, params.Partitions, 2)
+	sc0, err := params.Partitions[0].Skipped.Count()
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), sc0)
+	sc1, err := params.Partitions[1].Skipped.Count()
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), sc1)
+}
+
+// Tests that V1_1 proofs on post worker
+func TestWindowPostV1P1NV20Worker(t *testing.T) {
+	kit.QuietMiningLogs()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	blocktime := 2 * time.Millisecond
+
+	client, miner, _, ens := kit.EnsembleWorker(t,
+		kit.GenesisNetworkVersion(network.Version20),
+		kit.ConstructorOpts(
+			node.Override(new(config.ProvingConfig), func() config.ProvingConfig {
+				c := config.DefaultStorageMiner()
+				c.Proving.DisableBuiltinWindowPoSt = true
+				return c.Proving
+			}),
+			node.Override(new(*wdpost.WindowPoStScheduler), modules.WindowPostScheduler(
+				config.DefaultStorageMiner().Fees,
+				config.ProvingConfig{
+					DisableBuiltinWindowPoSt:  true,
+					DisableBuiltinWinningPoSt: false,
+					DisableWDPoStPreChecks:    false,
+				},
+			))),
+		kit.ThroughRPC(),
+		kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}))
+
+	ens.InterconnectAll().BeginMining(blocktime)
+
+	maddr, err := miner.ActorAddress(ctx)
+	require.NoError(t, err)
+
+	mi, err := client.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+	require.NoError(t, err)
+
+	wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
+	require.NoError(t, err)
+	en := wact.Nonce
+
+	// wait for a new message to be sent from worker address, it will be a PoSt
+
+waitForProof:
+	for {
+		//stm: @CHAIN_STATE_GET_ACTOR_001
+		wact, err := client.StateGetActor(ctx, mi.Worker, types.EmptyTSK)
+		require.NoError(t, err)
+		if wact.Nonce > en {
+			break waitForProof
+		}
+
+		build.Clock.Sleep(blocktime)
+	}
+
+	slm, err := client.StateListMessages(ctx, &api.MessageMatch{To: maddr}, types.EmptyTSK, 0)
+	require.NoError(t, err)
+
+	pmr, err := client.StateSearchMsg(ctx, types.EmptyTSK, slm[0], -1, false)
+	require.NoError(t, err)
+
+	nv, err := client.StateNetworkVersion(ctx, pmr.TipSet)
+	require.NoError(t, err)
+	require.Equal(t, network.Version20, nv)
+
+	require.True(t, pmr.Receipt.ExitCode.IsSuccess())
+
+	slmsg, err := client.ChainGetMessage(ctx, slm[0])
+	require.NoError(t, err)
+
+	var params miner11.SubmitWindowedPoStParams
+	require.NoError(t, params.UnmarshalCBOR(bytes.NewBuffer(slmsg.Params)))
+	require.Equal(t, abi.RegisteredPoStProof_StackedDrgWindow2KiBV1_1, params.Proofs[0].PoStProof)
+}

From 70d2899ead8815bb5366c9cf8bbe5bdcfcbf6bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Magiera?= <magik6k@gmail.com>
Date: Tue, 9 May 2023 19:11:15 +0200
Subject: [PATCH 04/46] itests: wdpost: Address review

---
 itests/worker_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/itests/worker_test.go b/itests/worker_test.go
index 87bb0047d..246c842c5 100644
--- a/itests/worker_test.go
+++ b/itests/worker_test.go
@@ -556,7 +556,7 @@ func TestWindowPostV1P1NV20WorkerFault(t *testing.T) {
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)
 
-	// wait for some crons(?)
+	// wait for sectors to be committed
 	require.Eventually(t, func() bool {
 		di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 		require.NoError(t, err)
@@ -596,11 +596,11 @@ func TestWindowPostV1P1NV20WorkerFault(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, uint64(2), n)
 
-		// Drop the sector in second partition
+		// Drop the sector in first partition
 		sid, err := secs.First()
 		require.NoError(t, err)
 
-		t.Logf("Drop sector %d; dl %d part %d", sid, di.Index+1, 0)
+		t.Logf("Drop sector %d; dl %d part %d", sid, di.Index, 0)
 
 		atomic.StoreUint64(&badsector, sid)
 		require.NoError(t, err)

From 4ca30abeef39714d9d684d03dae406bbf238448b Mon Sep 17 00:00:00 2001
From: Fridrik Asmundsson <fridrik01@gmail.com>
Date: Fri, 28 Apr 2023 10:16:55 +0000
Subject: [PATCH 05/46] Add support for blockHash param in eth_getLogs

---
 node/impl/full/eth.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/impl/full/eth.go b/node/impl/full/eth.go
index a26241cd7..2f1b30fad 100644
--- a/node/impl/full/eth.go
+++ b/node/impl/full/eth.go
@@ -1209,7 +1209,7 @@ func (e *EthEvent) installEthFilterSpec(ctx context.Context, filterSpec *ethtype
 			return nil, xerrors.Errorf("must not specify block hash and from/to block")
 		}
 
-		// TODO: derive a tipset hash from eth hash - might need to push this down into the EventFilterManager
+		tipsetCid = filterSpec.BlockHash.ToCid()
 	} else {
 		if filterSpec.FromBlock == nil || *filterSpec.FromBlock == "latest" {
 			ts := e.Chain.GetHeaviestTipSet()

From dbb892d89f9abc83a438a87487c5ff13d1a3ea50 Mon Sep 17 00:00:00 2001
From: Maciej Witowski <maciej.witowski@protocol.ai>
Date: Fri, 28 Apr 2023 17:01:11 +0200
Subject: [PATCH 06/46] lotus-fountain: make compatible with 0x addresses
 #10560

---
 cmd/lotus-fountain/main.go | 38 ++++++++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index 780aef916..464251cd3 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	rice "github.com/GeertJohan/go.rice"
@@ -19,6 +20,7 @@ import (
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 	lcli "github.com/filecoin-project/lotus/cli"
 )
 
@@ -193,14 +195,34 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	to, err := address.NewFromString(r.FormValue("address"))
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-	if to == address.Undef {
-		http.Error(w, "empty address", http.StatusBadRequest)
-		return
+	var to address.Address
+
+	addressInput := r.FormValue("address")
+	if strings.HasPrefix(addressInput, "0x") {
+		ethAddress, err := ethtypes.ParseEthAddress(addressInput)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		filecoinAddress, err := ethAddress.ToFilecoinAddress()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		to = filecoinAddress
+	} else {
+		filecoinAddress, err := address.NewFromString(addressInput)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		if filecoinAddress == address.Undef {
+			http.Error(w, "empty address", http.StatusBadRequest)
+			return
+		}
+
+		to = filecoinAddress
 	}
 
 	// Limit based on wallet address

From 08e6e041454785320d4372ac94fddd680eca293e Mon Sep 17 00:00:00 2001
From: Maciej Witowski <maciej.witowski@protocol.ai>
Date: Fri, 5 May 2023 13:16:48 +0200
Subject: [PATCH 07/46] Unify error handling

---
 cmd/lotus-fountain/main.go | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index 464251cd3..68f3d0e99 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -195,9 +195,11 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var to address.Address
-
 	addressInput := r.FormValue("address")
+
+	var filecoinAddress address.Address
+	var decodeError error
+
 	if strings.HasPrefix(addressInput, "0x") {
 		ethAddress, err := ethtypes.ParseEthAddress(addressInput)
 		if err != nil {
@@ -205,28 +207,22 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		filecoinAddress, err := ethAddress.ToFilecoinAddress()
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		to = filecoinAddress
+		filecoinAddress, decodeError = ethAddress.ToFilecoinAddress()
 	} else {
-		filecoinAddress, err := address.NewFromString(addressInput)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		if filecoinAddress == address.Undef {
-			http.Error(w, "empty address", http.StatusBadRequest)
-			return
-		}
+		filecoinAddress, decodeError = address.NewFromString(addressInput)
+	}
 
-		to = filecoinAddress
+	if decodeError != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if filecoinAddress == address.Undef {
+		http.Error(w, "empty address", http.StatusBadRequest)
+		return
 	}
 
 	// Limit based on wallet address
-	limiter := h.limiter.GetWalletLimiter(to.String())
+	limiter := h.limiter.GetWalletLimiter(filecoinAddress.String())
 	if !limiter.Allow() {
 		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": wallet limit", http.StatusTooManyRequests)
 		return
@@ -252,7 +248,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
 		Value: types.BigInt(h.sendPerRequest),
 		From:  h.from,
-		To:    to,
+		To:    filecoinAddress,
 	}, nil)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)

From 0ec3d442766e8476d85a65bf5381c80824dc9cb0 Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Wed, 10 May 2023 23:30:43 -0700
Subject: [PATCH 08/46] add grant-datacap support for lotus fountain

---
 cmd/lotus-fountain/main.go           | 60 +++++++++++++++++++++++++---
 cmd/lotus-fountain/site/datacap.html | 40 +++++++++++++++++++
 cmd/lotus-fountain/site/index.html   |  6 +++
 3 files changed, 101 insertions(+), 5 deletions(-)
 create mode 100644 cmd/lotus-fountain/site/datacap.html

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index 780aef916..d0e1964c4 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	verifregtypes9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"html/template"
 	"net"
 	"net/http"
@@ -18,6 +20,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 )
@@ -70,6 +73,11 @@ var runCmd = &cli.Command{
 			EnvVars: []string{"LOTUS_FOUNTAIN_AMOUNT"},
 			Value:   "50",
 		},
+		&cli.Uint64Flag{
+			Name:    "data-cap",
+			EnvVars: []string{"LOTUS_DATACAP_AMOUNT"},
+			Value:   10240,
+		},
 		&cli.Float64Flag{
 			Name:  "captcha-threshold",
 			Value: 0.5,
@@ -108,6 +116,7 @@ var runCmd = &cli.Command{
 			ctx:            ctx,
 			api:            nodeApi,
 			from:           from,
+			allowance:      types.NewInt(cctx.Uint64("data-cap")),
 			sendPerRequest: sendPerRequest,
 			limiter: NewLimiter(LimiterConfig{
 				TotalRate:   500 * time.Millisecond,
@@ -124,6 +133,8 @@ var runCmd = &cli.Command{
 		http.Handle("/", http.FileServer(box.HTTPBox()))
 		http.HandleFunc("/funds.html", prepFundsHtml(box))
 		http.Handle("/send", h)
+		http.HandleFunc("/datacap.html", prepDataCapHtml(box))
+		http.Handle("/datacap", h)
 		fmt.Printf("Open http://%s\n", cctx.String("front"))
 
 		go func() {
@@ -156,12 +167,24 @@ func prepFundsHtml(box *rice.Box) http.HandlerFunc {
 	}
 }
 
+func prepDataCapHtml(box *rice.Box) http.HandlerFunc {
+	tmpl := template.Must(template.New("datacaps").Parse(box.MustString("datacap.html")))
+	return func(w http.ResponseWriter, r *http.Request) {
+		err := tmpl.Execute(w, os.Getenv("RECAPTCHA_SITE_KEY"))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+	}
+}
+
 type handler struct {
 	ctx context.Context
 	api v0api.FullNode
 
 	from           address.Address
 	sendPerRequest types.FIL
+	allowance      types.BigInt
 
 	limiter        *Limiter
 	recapThreshold float64
@@ -187,6 +210,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusBadGateway)
 		return
 	}
+
 	if !capResp.Success || capResp.Score < h.recapThreshold {
 		log.Infow("spam", "capResp", capResp)
 		http.Error(w, "spam protection", http.StatusUnprocessableEntity)
@@ -227,11 +251,37 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
-		Value: types.BigInt(h.sendPerRequest),
-		From:  h.from,
-		To:    to,
-	}, nil)
+	var smsg *types.SignedMessage
+	if r.RequestURI == "/send" {
+		smsg, err = h.api.MpoolPushMessage(
+			h.ctx, &types.Message{
+				Value: types.BigInt(h.sendPerRequest),
+				From:  h.from,
+				To:    to,
+			}, nil)
+	} else if r.RequestURI == "/datacap" {
+		var params []byte
+		params, err = actors.SerializeParams(
+			&verifregtypes9.AddVerifiedClientParams{
+				Address:   to,
+				Allowance: h.allowance,
+			})
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		smsg, err = h.api.MpoolPushMessage(
+			h.ctx, &types.Message{
+				Params: params,
+				From:   h.from,
+				To:     verifreg.Address,
+				Method: verifreg.Methods.AddVerifiedClient,
+			}, nil)
+	} else {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
diff --git a/cmd/lotus-fountain/site/datacap.html b/cmd/lotus-fountain/site/datacap.html
new file mode 100644
index 000000000..10cf1f411
--- /dev/null
+++ b/cmd/lotus-fountain/site/datacap.html
@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Grant DataCap - Lotus Fountain</title>
+    <link rel="stylesheet" type="text/css" href="main.css">
+	<script src="https://www.google.com/recaptcha/api.js"></script>
+	<script>
+   		function onSubmit(token) {
+     	document.getElementById("funds-form").submit();
+   	}
+	</script>
+
+</head>
+<body>
+<div class="Index">
+    <div class="Index-nodes">
+        <div class="Index-node">
+            [GRANT DATACAP]
+        </div>
+        <div class="Index-node">
+            <form action='/datacap' method='post' id='datacap-form'>
+                <span>Enter destination address:</span>
+                <input type='text' name='address' style="width: 300px">
+                <button class="g-recaptcha"
+                        data-sitekey="{{ . }}"
+                        data-callback='onSubmit'
+                        data-action='submit'>Grant Datacap</button>
+            </form>
+        </div>
+    </div>
+
+    <div class="Index-footer">
+        <div>
+            <a href="index.html">[Back]</a>
+            <span style="float: right">Not dispensing real Filecoin tokens</span>
+        </div>
+    </div>
+</div>
+</body>
+</html>
diff --git a/cmd/lotus-fountain/site/index.html b/cmd/lotus-fountain/site/index.html
index 644960225..4dc3df478 100644
--- a/cmd/lotus-fountain/site/index.html
+++ b/cmd/lotus-fountain/site/index.html
@@ -13,6 +13,12 @@
         <div class="Index-node">
             <a href="funds.html">[Send Funds]</a>
         </div>
+        <div class="Index-node">
+            [LOTUS DEVNET GRANT DATACAP]
+        </div>
+        <div class="Index-node">
+            <a href="datacap.html">[Grant DataCap]</a>
+        </div>
     </div>
     <div class="Index-footer">
         <div>

From 96c1ab413fd285681a0ba31a5fc6fe087d91414e Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Thu, 11 May 2023 00:06:05 -0700
Subject: [PATCH 09/46] run gofmt to clean the imports error

---
 cmd/lotus-fountain/main.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index d0e1964c4..14968cb72 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -3,8 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	verifregtypes9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
-	"github.com/filecoin-project/lotus/chain/actors"
 	"html/template"
 	"net"
 	"net/http"
@@ -17,9 +15,11 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	verifregtypes9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"

From f4a3207ede353f55e51204ac4ebfa2279a785003 Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Thu, 11 May 2023 09:32:40 -0400
Subject: [PATCH 10/46] fix: splitstore: remove deadlock around waiting for
 sync

---
 blockstore/splitstore/splitstore_compact.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/blockstore/splitstore/splitstore_compact.go b/blockstore/splitstore/splitstore_compact.go
index f96f9d370..534565bf3 100644
--- a/blockstore/splitstore/splitstore_compact.go
+++ b/blockstore/splitstore/splitstore_compact.go
@@ -1114,13 +1114,17 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs abi.ChainEp
 					if err := walkBlock(c); err != nil {
 						return xerrors.Errorf("error walking block (cid: %s): %w", c, err)
 					}
+
+					if err := s.checkYield(); err != nil {
+						return xerrors.Errorf("check yield: %w", err)
+					}
 				}
 				return nil
 			})
 		}
 
 		if err := g.Wait(); err != nil {
-			return err
+			return xerrors.Errorf("walkBlock workers errored: %w", err)
 		}
 	}
 
@@ -1153,8 +1157,8 @@ func (s *SplitStore) walkObject(c cid.Cid, visitor ObjectVisitor, f func(cid.Cid
 	}
 
 	// check this before recursing
-	if err := s.checkYield(); err != nil {
-		return 0, err
+	if err := s.checkClosing(); err != nil {
+		return 0, xerrors.Errorf("check closing: %w", err)
 	}
 
 	var links []cid.Cid
@@ -1222,8 +1226,8 @@ func (s *SplitStore) walkObjectIncomplete(c cid.Cid, visitor ObjectVisitor, f, m
 	}
 
 	// check this before recursing
-	if err := s.checkYield(); err != nil {
-		return sz, err
+	if err := s.checkClosing(); err != nil {
+		return sz, xerrors.Errorf("check closing: %w", err)
 	}
 
 	var links []cid.Cid

From f39478140a7e792dfc9a84506907402242efe2c6 Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Thu, 11 May 2023 15:08:12 -0400
Subject: [PATCH 11/46] feat: deflake sector_import_simple

---
 itests/sector_import_simple_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/itests/sector_import_simple_test.go b/itests/sector_import_simple_test.go
index 0cf7e098b..f34438694 100644
--- a/itests/sector_import_simple_test.go
+++ b/itests/sector_import_simple_test.go
@@ -39,7 +39,9 @@ func TestSectorImportAfterPC2(t *testing.T) {
 	////////
 	// Start a miner node
 
-	client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC())
+	// We use two miners so that in case the actively tested miner misses PoSt, we still have a blockchain
+	client, miner, _, ens := kit.EnsembleOneTwo(t, kit.ThroughRPC())
+
 	ens.InterconnectAll().BeginMining(blockTime)
 
 	ctx := context.Background()

From f671654bdd782d82a2dc0392ab54dbc74feeefac Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Fri, 12 May 2023 16:35:00 +0200
Subject: [PATCH 12/46] Make redeclare cmd work

- Fix an issue where `lotus-miner storage redeclare --all` required an argument
- Actually implement the logic to redeclare a single sectors in a single storage path
---
 cmd/lotus-miner/storage.go | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-miner/storage.go b/cmd/lotus-miner/storage.go
index 89353497d..761f070f0 100644
--- a/cmd/lotus-miner/storage.go
+++ b/cmd/lotus-miner/storage.go
@@ -235,14 +235,19 @@ var storageRedeclareCmd = &cli.Command{
 		defer closer()
 		ctx := lcli.ReqContext(cctx)
 
-		if cctx.NArg() != 1 {
-			return lcli.IncorrectNumArgs(cctx)
+		// check if no argument and no --id or --all flag is provided
+		if cctx.NArg() == 0 && !cctx.IsSet("id") && !cctx.Bool("all") {
+			return xerrors.Errorf("You must specify a storage path, or --id, or --all")
 		}
 
 		if cctx.IsSet("id") && cctx.Bool("all") {
 			return xerrors.Errorf("--id and --all can't be passed at the same time")
 		}
 
+		if cctx.Bool("all") && cctx.NArg() > 0 {
+			return xerrors.Errorf("No additional arguments are expected when --all is set")
+		}
+
 		if cctx.IsSet("id") {
 			id := storiface.ID(cctx.String("id"))
 			return minerApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
@@ -252,7 +257,24 @@ var storageRedeclareCmd = &cli.Command{
 			return minerApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
 		}
 
-		return xerrors.Errorf("either --all or --id must be specified")
+		// As no --id or --all flag is set, we can assume the argument is a path.
+		path := cctx.Args().First()
+		metaFilePath := filepath.Join(path, "sectorstore.json")
+
+		var meta storiface.LocalStorageMeta
+		metaFile, err := os.Open(metaFilePath)
+		if err != nil {
+			return xerrors.Errorf("Failed to open meta file: %w", err)
+		}
+		defer metaFile.Close()
+
+		err = json.NewDecoder(metaFile).Decode(&meta)
+		if err != nil {
+			return xerrors.Errorf("Failed to decode meta file: %w", err)
+		}
+
+		id := meta.ID
+		return minerApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
 	},
 }
 

From 760a27dd4b6c1668b406b31a579e40eb57a05fdb Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Fri, 12 May 2023 11:04:55 -0400
Subject: [PATCH 13/46] feat: itests: add logs to blockminer.go failure case

---
 itests/kit/blockminer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/itests/kit/blockminer.go b/itests/kit/blockminer.go
index cbe352dd5..02876ebb3 100644
--- a/itests/kit/blockminer.go
+++ b/itests/kit/blockminer.go
@@ -236,7 +236,7 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 					break
 				}
 
-				require.NotEqual(bm.t, i, nloops-1, "block never managed to sync to node")
+				require.NotEqual(bm.t, i, nloops-1, "block at height %d never managed to sync to node, which is at height %d", target, ts.Height())
 				time.Sleep(time.Millisecond * 10)
 			}
 
@@ -348,7 +348,7 @@ func (bm *BlockMiner) MineUntilBlock(ctx context.Context, fn *TestFullNode, cb f
 					break
 				}
 
-				require.NotEqual(bm.t, i, nloops-1, "block never managed to sync to node")
+				require.NotEqual(bm.t, i, nloops-1, "block at height %d never managed to sync to node, which is at height %d", epoch, ts.Height())
 				time.Sleep(time.Millisecond * 10)
 			}
 

From 9486aa98dbd49a7da39f69111310e3e7b44990f4 Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Fri, 12 May 2023 10:48:18 -0700
Subject: [PATCH 14/46] fixed the merged break for
 to->cmd/lotus-fountain/main.go

---
 cmd/lotus-fountain/main.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index fe417dddb..2e61fcb6d 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -275,13 +275,13 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			h.ctx, &types.Message{
 				Value: types.BigInt(h.sendPerRequest),
 				From:  h.from,
-				To:    to,
+				To:    filecoinAddress,
 			}, nil)
 	} else if r.RequestURI == "/datacap" {
 		var params []byte
 		params, err = actors.SerializeParams(
 			&verifregtypes9.AddVerifiedClientParams{
-				Address:   to,
+				Address:   filecoinAddress,
 				Allowance: h.allowance,
 			})
 		if err != nil {
@@ -300,7 +300,6 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return

From 4f3b6175c00b0af4259a5f7345509b05e342968d Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Fri, 12 May 2023 10:59:28 -0700
Subject: [PATCH 15/46] follow the new style for html from master branch

---
 cmd/lotus-fountain/site/datacap.html | 2 +-
 cmd/lotus-fountain/site/index.html   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/lotus-fountain/site/datacap.html b/cmd/lotus-fountain/site/datacap.html
index 10cf1f411..f42a3a55f 100644
--- a/cmd/lotus-fountain/site/datacap.html
+++ b/cmd/lotus-fountain/site/datacap.html
@@ -15,7 +15,7 @@
 <div class="Index">
     <div class="Index-nodes">
         <div class="Index-node">
-            [GRANT DATACAP]
+            GRANT DATACAP
         </div>
         <div class="Index-node">
             <form action='/datacap' method='post' id='datacap-form'>
diff --git a/cmd/lotus-fountain/site/index.html b/cmd/lotus-fountain/site/index.html
index 5398cf4f1..379267f1f 100644
--- a/cmd/lotus-fountain/site/index.html
+++ b/cmd/lotus-fountain/site/index.html
@@ -14,10 +14,10 @@
             <a href="funds.html">Send Funds</a>
         </div>
         <div class="Index-node">
-            [LOTUS DEVNET GRANT DATACAP]
+            LOTUS DEVNET GRANT DATACAP
         </div>
         <div class="Index-node">
-            <a href="datacap.html">[Grant DataCap]</a>
+            <a href="datacap.html">Grant DataCap</a>
         </div>
     </div>
     <div class="Index-footer">

From 0d7d906abf858bebeaa6b4ba6ca828d7a65e6fbc Mon Sep 17 00:00:00 2001
From: Yolan Romailler <anomalroil@users.noreply.github.com>
Date: Fri, 12 May 2023 21:26:26 +0200
Subject: [PATCH 16/46] Adding extra data check on DecodeBlockMsg

fix: types: error out on decoding BlockMsg with extraneous data
Fixes OSS-fuzz issue 48208: lotus:fuzz_block_msg

Signed-off-by: Yolan Romailler <anomalroil@users.noreply.github.com>
---
 chain/types/blockmsg.go      |  8 ++++++--
 chain/types/blockmsg_test.go | 40 ++++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 chain/types/blockmsg_test.go

diff --git a/chain/types/blockmsg.go b/chain/types/blockmsg.go
index f3114499d..f8f0a08db 100644
--- a/chain/types/blockmsg.go
+++ b/chain/types/blockmsg.go
@@ -2,6 +2,7 @@ package types
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ipfs/go-cid"
 )
@@ -14,10 +15,13 @@ type BlockMsg struct {
 
 func DecodeBlockMsg(b []byte) (*BlockMsg, error) {
 	var bm BlockMsg
-	if err := bm.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
+	data := bytes.NewReader(b)
+	if err := bm.UnmarshalCBOR(data); err != nil {
 		return nil, err
 	}
-
+	if l := data.Len(); l != 0 {
+		return nil, fmt.Errorf("extraneous data in BlockMsg CBOR encoding: got %d unexpected bytes", l)
+	}
 	return &bm, nil
 }
 
diff --git a/chain/types/blockmsg_test.go b/chain/types/blockmsg_test.go
new file mode 100644
index 000000000..20cbe324d
--- /dev/null
+++ b/chain/types/blockmsg_test.go
@@ -0,0 +1,40 @@
+package types
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeBlockMsg(t *testing.T) {
+	type args struct {
+		b []byte
+	}
+	tests := []struct {
+		name    string
+		data    []byte
+		want    *BlockMsg
+		wantErr bool
+	}{
+		{"decode empty BlockMsg with extra data at the end", []byte{0x83, 0xf6, 0x80, 0x80, 0x20}, new(BlockMsg), true},
+		{"decode valid empty BlockMsg", []byte{0x83, 0xf6, 0x80, 0x80}, new(BlockMsg), false},
+		{"decode invalid cbor", []byte{0x83, 0xf6, 0x80}, nil, true},
+	}
+	for _, tt := range tests {
+		data := tt.data
+		want := tt.want
+		wantErr := tt.wantErr
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DecodeBlockMsg(data)
+			if wantErr {
+				assert.Errorf(t, err, "DecodeBlockMsg(%v)", data)
+				return
+			}
+			assert.NoErrorf(t, err, "DecodeBlockMsg(%v)", data)
+			assert.Equalf(t, want, got, "DecodeBlockMsg(%v)", data)
+			serialized, err := got.Serialize()
+			assert.NoErrorf(t, err, "DecodeBlockMsg(%v)", data)
+			assert.Equalf(t, serialized, data, "DecodeBlockMsg(%v)", data)
+		})
+	}
+}

From af0303ff7d683481fa35f3b1697a770deccd3430 Mon Sep 17 00:00:00 2001
From: Jacob Heun <jacob.heun@gmail.com>
Date: Wed, 3 May 2023 16:17:00 +0200
Subject: [PATCH 17/46] chore: update go-fil-markets

---
 go.mod |  9 +++++----
 go.sum | 18 ++++++++++--------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/go.mod b/go.mod
index 50f9b40b8..f97f57fc3 100644
--- a/go.mod
+++ b/go.mod
@@ -38,10 +38,10 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.1
 	github.com/filecoin-project/go-commp-utils v0.1.3
 	github.com/filecoin-project/go-crypto v0.0.1
-	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4
+	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.27.0-rc1
+	github.com/filecoin-project/go-fil-markets v1.28.1
 	github.com/filecoin-project/go-jsonrpc v0.3.1
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
@@ -87,7 +87,7 @@ require (
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ds-measure v0.2.0
 	github.com/ipfs/go-fs-lock v0.0.7
-	github.com/ipfs/go-graphsync v0.14.3
+	github.com/ipfs/go-graphsync v0.14.5
 	github.com/ipfs/go-ipfs-blockstore v1.3.0
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.5
@@ -111,7 +111,7 @@ require (
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
-	github.com/ipni/index-provider v0.11.0
+	github.com/ipni/index-provider v0.12.0
 	github.com/ipni/storetheindex v0.5.10
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
@@ -249,6 +249,7 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
+	github.com/ipni/go-libipni v0.0.7 // indirect
 	github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
diff --git a/go.sum b/go.sum
index b895ba0ef..9665d04be 100644
--- a/go.sum
+++ b/go.sum
@@ -312,8 +312,8 @@ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o=
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 h1:Y5RMvFT4OthsAhDx7xKfkJ5QdiWq9Ox7N46Mi0PF0PE=
-github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4/go.mod h1:1WDoUgWYB2KvogfPTdDmoyBTa9cb9+oGwqKCCipnJeY=
+github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6 h1:EsbXTWsBKT764qtX4MMJBNXMHoEa+g5Xg01azMqxXX0=
+github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6/go.mod h1:cX1acvFVWC5EXnnmFPWEFXbO7nLUdSZa+nqgi1QpTpw=
 github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w=
 github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
@@ -321,8 +321,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88Oq
 github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
-github.com/filecoin-project/go-fil-markets v1.27.0-rc1 h1:SYXKFONg6IaQlmXEqAEaPtb/xX57dZYMkx3LwjsLr7w=
-github.com/filecoin-project/go-fil-markets v1.27.0-rc1/go.mod h1:JkW4rU0+LqfO/DLi/wyR58AqxaUdnlSofWeu8un2XLU=
+github.com/filecoin-project/go-fil-markets v1.28.1 h1:2iq1o/ZY1RSbVU8OX7nwWO66SYtdjKEngshc4Jeua/Y=
+github.com/filecoin-project/go-fil-markets v1.28.1/go.mod h1:qy9LNu9t77I184VB6Pa4WKRtGfB8Vl0t8zfOLHkDqWY=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -738,8 +738,8 @@ github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHL
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
 github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
 github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
-github.com/ipfs/go-graphsync v0.14.3 h1:IXH9S7AraMQ0J6Fzcl8rqSPqLn+es33bD8OW2KNyU/o=
-github.com/ipfs/go-graphsync v0.14.3/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo=
+github.com/ipfs/go-graphsync v0.14.5 h1:SKQog4ZABe+yy7OtTsrMoSQfFmEPPi0qP5sl6bFN8xM=
+github.com/ipfs/go-graphsync v0.14.5/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
 github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
@@ -885,8 +885,10 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
-github.com/ipni/index-provider v0.11.0 h1:q2PdK6JpYB9bzlntfkRYNBjhg4Qtko5+iXRonO88TAg=
-github.com/ipni/index-provider v0.11.0/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
+github.com/ipni/go-libipni v0.0.7 h1:L0AnFQagedfJU5mJ7kz0H8P1452brJveOQeS6p3MmbA=
+github.com/ipni/go-libipni v0.0.7/go.mod h1:TlGZaGMGIVpeb6fiwttfY1JgaMnH+HDVPzxgRJJPaQY=
+github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwSD8jhQ=
+github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4=
 github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
 github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=

From a29e8226c77a8d867c463920484924439979c869 Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Mon, 15 May 2023 11:51:43 +0200
Subject: [PATCH 18/46] Set drop-missing to true

Set drop-missing to true
---
 cmd/lotus-miner/storage.go          | 11 ++++++++---
 documentation/en/cli-lotus-miner.md |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/cmd/lotus-miner/storage.go b/cmd/lotus-miner/storage.go
index 761f070f0..fdd5b5696 100644
--- a/cmd/lotus-miner/storage.go
+++ b/cmd/lotus-miner/storage.go
@@ -225,6 +225,7 @@ var storageRedeclareCmd = &cli.Command{
 		&cli.BoolFlag{
 			Name:  "drop-missing",
 			Usage: "Drop index entries with missing files",
+			Value: true,
 		},
 	},
 	Action: func(cctx *cli.Context) error {
@@ -264,13 +265,17 @@ var storageRedeclareCmd = &cli.Command{
 		var meta storiface.LocalStorageMeta
 		metaFile, err := os.Open(metaFilePath)
 		if err != nil {
-			return xerrors.Errorf("Failed to open meta file: %w", err)
+			return xerrors.Errorf("Failed to open file: %w", err)
 		}
-		defer metaFile.Close()
+		defer func() {
+			if closeErr := metaFile.Close(); closeErr != nil {
+				log.Error("Failed to close the file: %v", closeErr)
+			}
+		}()
 
 		err = json.NewDecoder(metaFile).Decode(&meta)
 		if err != nil {
-			return xerrors.Errorf("Failed to decode meta file: %w", err)
+			return xerrors.Errorf("Failed to decode file: %w", err)
 		}
 
 		id := meta.ID
diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md
index 7c646a3b4..1446a36a9 100644
--- a/documentation/en/cli-lotus-miner.md
+++ b/documentation/en/cli-lotus-miner.md
@@ -1285,7 +1285,7 @@ USAGE:
 
 OPTIONS:
    --all           redeclare all storage paths (default: false)
-   --drop-missing  Drop index entries with missing files (default: false)
+   --drop-missing  Drop index entries with missing files (default: true)
    --id value      storage path ID
    
 ```

From 7519f9fdf1ddb4394ba3f8d0db359fdcfc29dce6 Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Mon, 15 May 2023 13:13:06 +0200
Subject: [PATCH 19/46] fix: Add lotus-miner legacy-markets alert

fix: Add lotus-miner legacy-markets alert
---
 node/builder.go        |  1 +
 node/builder_miner.go  |  4 ++++
 node/modules/alerts.go | 10 ++++++++++
 3 files changed, 15 insertions(+)

diff --git a/node/builder.go b/node/builder.go
index 75162f5ea..af604983d 100644
--- a/node/builder.go
+++ b/node/builder.go
@@ -89,6 +89,7 @@ const (
 
 	// health checks
 	CheckFDLimit
+	LegacyMarketsEOL
 
 	// libp2p
 	PstoreAddSelfKeysKey
diff --git a/node/builder_miner.go b/node/builder_miner.go
index 68d5a32f8..d9c92422d 100644
--- a/node/builder_miner.go
+++ b/node/builder_miner.go
@@ -147,6 +147,10 @@ func ConfigStorageMiner(c interface{}) Option {
 		),
 
 		If(cfg.Subsystems.EnableMarkets,
+
+			// Alert that legacy-markets is being deprecated
+			Override(LegacyMarketsEOL, modules.LegacyMarketsEOL),
+
 			// Markets
 			Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
 			Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(cfg.Dealmaking.SimultaneousTransfersForStorage, cfg.Dealmaking.SimultaneousTransfersForStoragePerClient, cfg.Dealmaking.SimultaneousTransfersForRetrieval)),
diff --git a/node/modules/alerts.go b/node/modules/alerts.go
index 75be08354..df6b76435 100644
--- a/node/modules/alerts.go
+++ b/node/modules/alerts.go
@@ -32,6 +32,16 @@ func CheckFdLimit(min uint64) func(al *alerting.Alerting) {
 	}
 }
 
+func LegacyMarketsEOL(al *alerting.Alerting) {
+	// Add alert if lotus-miner legacy markets subsystem is still in use
+	alert := al.AddAlertType("system", "EOL")
+
+	// Alert with a message to migrate to Boost or similar markets subsystems
+	al.Raise(alert, map[string]string{
+		"message": "The lotus-miner legacy markets subsystem is deprecated and will be removed in a future release. Please migrate to [Boost](https://boost.filecoin.io) or similar markets subsystems.",
+	})
+}
+
 // TODO: More things:
 //  * Space in repo dirs (taking into account mounts)
 //  * Miner

From 01317818080a5647276246694902d68db0299d92 Mon Sep 17 00:00:00 2001
From: Fridrik Asmundsson <fridrik01@gmail.com>
Date: Tue, 16 May 2023 11:25:57 +0000
Subject: [PATCH 20/46] feat: run lotus-shed commands in context that is
 cancelled on sigterm

---
 cmd/lotus-shed/main.go | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go
index 9cb1cd32c..6aebb264f 100644
--- a/cmd/lotus-shed/main.go
+++ b/cmd/lotus-shed/main.go
@@ -1,8 +1,10 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
 
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
@@ -118,7 +120,20 @@ func main() {
 		},
 	}
 
-	if err := app.Run(os.Args); err != nil {
+	// terminate early on ctrl+c
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-c
+		cancel()
+		fmt.Println("Received interrupt, stopping... Press CTRL+C again to force stop")
+		<-c
+		fmt.Println("Forcing stop")
+		os.Exit(1)
+	}()
+
+	if err := app.RunContext(ctx, os.Args); err != nil {
 		log.Errorf("%+v", err)
 		os.Exit(1)
 		return

From b0d1385273557e1842474adc445a253ece959286 Mon Sep 17 00:00:00 2001
From: Shrenuj Bansal <shrenuj.bansal@protocol.ai>
Date: Tue, 16 May 2023 21:19:03 -0400
Subject: [PATCH 21/46] Check if epoch is negative in GetTipsetByHeight

---
 chain/store/store.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/chain/store/store.go b/chain/store/store.go
index fd12b88a5..88103ac48 100644
--- a/chain/store/store.go
+++ b/chain/store/store.go
@@ -1161,6 +1161,10 @@ func (cs *ChainStore) TryFillTipSet(ctx context.Context, ts *types.TipSet) (*Ful
 // selects the tipset before the null round if true, and the tipset following
 // the null round if false.
 func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, ts *types.TipSet, prev bool) (*types.TipSet, error) {
+	if h < 0 {
+		return nil, xerrors.Errorf("height %d is negative", h)
+	}
+
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}

From c2e2725855b4c6f47c471700025983df476ea608 Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Mon, 24 Apr 2023 17:34:56 +0200
Subject: [PATCH 22/46] update traces, es transport with batches and fasthttp

---
 go.mod                                        |   2 +
 go.sum                                        |   4 +
 node/modules/lp2p/pubsub.go                   | 204 ++++++++++++++----
 .../modules/tracer/elasticsearch_transport.go |  65 +++---
 node/modules/tracer/fasthttp_transport.go     |  74 +++++++
 5 files changed, 276 insertions(+), 73 deletions(-)
 create mode 100644 node/modules/tracer/fasthttp_transport.go

diff --git a/go.mod b/go.mod
index f97f57fc3..1b091e095 100644
--- a/go.mod
+++ b/go.mod
@@ -144,6 +144,7 @@ require (
 	github.com/stretchr/testify v1.8.2
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/urfave/cli/v2 v2.16.3
+	github.com/valyala/fasthttp v1.45.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
@@ -179,6 +180,7 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akavel/rsrc v0.8.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
diff --git a/go.sum b/go.sum
index 9665d04be..4b4dd2a9f 100644
--- a/go.sum
+++ b/go.sum
@@ -93,6 +93,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -1634,6 +1636,8 @@ github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk=
 github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.45.0 h1:zPkkzpIn8tdHZUrVa6PzYd0i5verqiPSkgTd3bSUcpA=
+github.com/valyala/fasthttp v1.45.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 5376d93c8..038c129ac 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -381,9 +381,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 	allowTopics = append(allowTopics, drandTopics...)
 	options = append(options,
 		pubsub.WithSubscriptionFilter(
-			pubsub.WrapLimitSubscriptionFilter(
-				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
-				100)))
+		pubsub.WrapLimitSubscriptionFilter(
+		pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
+		100)))
 
 	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracer != "" {
@@ -394,6 +394,20 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 
 		transports = append(transports, jsonTransport)
 	}
+
+	tps := make([]string, 0) // range of topics that will be submited to the traces
+	addTopicToList := func(topicList []string, newTopic string) []string {
+		// check if the topic is already in the list
+		for _, tp := range topicList {
+			if tp == newTopic {
+				return topicList
+			}
+		}
+		// add it otherwise
+		return append(topicList, newTopic)
+	}
+
+	// tracer
 	if in.Cfg.ElasticSearchTracer != "" {
 		elasticSearchTransport, err := tracer.NewElasticSearchTransport(
 			in.Cfg.ElasticSearchTracer,
@@ -403,7 +417,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 			return nil, err
 		}
 		transports = append(transports, elasticSearchTransport)
+		tps = addTopicToList(tps, build.BlocksTopic(in.Nn))
 	}
+
 	lt := tracer.NewLotusTracer(transports, in.Host.ID(), in.Cfg.TracerSourceAuth)
 
 	// tracer
@@ -412,28 +428,25 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 		if err != nil {
 			return nil, err
 		}
-
 		pi, err := peer.AddrInfoFromP2pAddr(a)
 		if err != nil {
 			return nil, err
 		}
-
 		tr, err := pubsub.NewRemoteTracer(context.TODO(), in.Host, *pi)
 		if err != nil {
 			return nil, err
 		}
-
 		pst := newPeerScoreTracker(lt, in.Sk)
-		trw := newTracerWrapper(tr, lt, build.BlocksTopic(in.Nn))
+		trw := newTracerWrapper(tr, lt, tps...)
 
 		options = append(options, pubsub.WithEventTracer(trw))
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	} else {
 		// still instantiate a tracer for collecting metrics
-		trw := newTracerWrapper(nil, lt)
-		options = append(options, pubsub.WithEventTracer(trw))
-
 		pst := newPeerScoreTracker(lt, in.Sk)
+		trw := newTracerWrapper(nil, lt, tps...)
+
+		options = append(options, pubsub.WithEventTracer(trw))
 		options = append(options, pubsub.WithPeerScoreInspect(pst.UpdatePeerScore, 10*time.Second))
 	}
 
@@ -457,7 +470,6 @@ func newTracerWrapper(
 			topicsMap[topic] = struct{}{}
 		}
 	}
-
 	return &tracerWrapper{lp2pTracer: lp2pTracer, lotusTracer: lotusTracer, topics: topicsMap}
 }
 
@@ -486,71 +498,169 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 			if trw.lp2pTracer != nil {
 				trw.lp2pTracer.Trace(evt)
 			}
-
 			if trw.lotusTracer != nil {
 				trw.lotusTracer.Trace(evt)
 			}
 		}
+
 	case pubsub_pb.TraceEvent_DELIVER_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1))
-		if trw.traceMessage(evt.GetDeliverMessage().GetTopic()) {
-			if trw.lp2pTracer != nil {
-				trw.lp2pTracer.Trace(evt)
-			}
-
-			if trw.lotusTracer != nil {
-				trw.lotusTracer.Trace(evt)
-			}
-		}
+		
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
 		if trw.traceMessage(evt.GetRejectMessage().GetTopic()) {
 			if trw.lp2pTracer != nil {
 				trw.lp2pTracer.Trace(evt)
 			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
+
+	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
+		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
+		if trw.traceMessage(evt.GetDuplicateMessage().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
+
+	case pubsub_pb.TraceEvent_JOIN:
+		if trw.traceMessage(evt.GetJoin().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
+	case pubsub_pb.TraceEvent_LEAVE:
+		if trw.traceMessage(evt.GetLeave().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
+
+	case pubsub_pb.TraceEvent_GRAFT:
+		if trw.traceMessage(evt.GetGraft().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
 
 			if trw.lotusTracer != nil {
 				trw.lotusTracer.Trace(evt)
 			}
 		}
-	case pubsub_pb.TraceEvent_DUPLICATE_MESSAGE:
-		stats.Record(context.TODO(), metrics.PubsubDuplicateMessage.M(1))
-	case pubsub_pb.TraceEvent_JOIN:
-		if trw.lp2pTracer != nil {
-			trw.lp2pTracer.Trace(evt)
-		}
 
-		if trw.lotusTracer != nil {
-			trw.lotusTracer.Trace(evt)
-		}
-	case pubsub_pb.TraceEvent_LEAVE:
-		if trw.lp2pTracer != nil {
-			trw.lp2pTracer.Trace(evt)
-		}
-
-		if trw.lotusTracer != nil {
-			trw.lotusTracer.Trace(evt)
-		}
-	case pubsub_pb.TraceEvent_GRAFT:
-		if trw.lp2pTracer != nil {
-			trw.lp2pTracer.Trace(evt)
-		}
-
-		if trw.lotusTracer != nil {
-			trw.lotusTracer.Trace(evt)
-		}
 	case pubsub_pb.TraceEvent_PRUNE:
+		if trw.traceMessage(evt.GetPrune().GetTopic()) {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
+
+	case pubsub_pb.TraceEvent_ADD_PEER:
 		if trw.lp2pTracer != nil {
 			trw.lp2pTracer.Trace(evt)
 		}
-
 		if trw.lotusTracer != nil {
 			trw.lotusTracer.Trace(evt)
 		}
+
+	case pubsub_pb.TraceEvent_REMOVE_PEER:
+		if trw.lp2pTracer != nil {
+			trw.lp2pTracer.Trace(evt)
+		}
+		if trw.lotusTracer != nil {
+			trw.lotusTracer.Trace(evt)
+		}
+
 	case pubsub_pb.TraceEvent_RECV_RPC:
 		stats.Record(context.TODO(), metrics.PubsubRecvRPC.M(1))
+		// only track the RPC Calls from IWANT / IHAVE / BLOCK topic
+		controlRPC := evt.GetRecvRPC().GetMeta().GetControl()
+		ihave := controlRPC.GetIhave()
+		iwant := controlRPC.GetIwant()
+		msgsRPC := evt.GetRecvRPC().GetMeta().GetMessages()
+
+		// check if any of the messages we are sending belong to a trackable topic
+		var validTopic bool = false
+		for _, topic := range msgsRPC {
+			if trw.traceMessage(topic.GetTopic()) {
+				validTopic = true
+				break
+			}
+		}
+		// track if the Iwant / Ihave messages are from a valid Topic
+		var validIhave bool = false
+		for _, msgs := range ihave {
+			if trw.traceMessage(msgs.GetTopic()) {
+				validIhave = true
+				break
+			}
+		}
+		// check if we have any of iwant msgs (it doesn't classify per topic - just msg.ID)
+		var validIwant bool = false
+		if len(iwant) > 0 {
+			validIwant = true
+		}
+
+		// trace the event if any of the flags was triggered
+		if validIhave || validIwant || validTopic {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
 	case pubsub_pb.TraceEvent_SEND_RPC:
 		stats.Record(context.TODO(), metrics.PubsubSendRPC.M(1))
+		// only track the RPC Calls from IWANT / IHAVE / BLOCK topic
+		controlRPC := evt.GetSendRPC().GetMeta().GetControl()
+		ihave := controlRPC.GetIhave()
+		iwant := controlRPC.GetIwant()
+		msgsRPC := evt.GetSendRPC().GetMeta().GetMessages()
+
+		// check if any of the messages we are sending belong to a trackable topic
+		var validTopic bool = false
+		for _, topic := range msgsRPC {
+			if trw.traceMessage(topic.GetTopic()) {
+				validTopic = true
+				break
+			}
+		}
+		// track if the Iwant / Ihave messages are from a valid Topic
+		var validIhave bool = false
+		for _, msgs := range ihave {
+			if trw.traceMessage(msgs.GetTopic()) {
+				validIhave = true
+				break
+			}
+		}
+		// check if there was any of the Iwant msgs
+		var validIwant bool = false
+		if len(iwant) > 0 {
+			validIwant = true
+		}
+		// trace the msgs if any of the flags was triggered
+		if validIhave || validIwant || validTopic {
+			if trw.lp2pTracer != nil {
+				trw.lp2pTracer.Trace(evt)
+			}
+			if trw.lotusTracer != nil {
+				trw.lotusTracer.Trace(evt)
+			}
+		}
 	case pubsub_pb.TraceEvent_DROP_RPC:
 		stats.Record(context.TODO(), metrics.PubsubDropRPC.M(1))
 	}
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 1f6f9a157..3cc50b388 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -4,15 +4,19 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"bytes"
 	"net/url"
-	"strings"
+	"time"
 
 	"github.com/elastic/go-elasticsearch/v7"
-	"github.com/elastic/go-elasticsearch/v7/esapi"
+	"github.com/elastic/go-elasticsearch/v7/esutil"
 )
 
 const (
 	ElasticSearchDefaultIndex = "lotus-pubsub"
+	flushInterval             = 10 * time.Second
+	flushBytes                = 1024 * 1024 // MB
+	esWorkers                 = 2           // TODO: hardcoded
 )
 
 func NewElasticSearchTransport(connectionString string, elasticsearchIndex string) (TracerTransport, error) {
@@ -30,10 +34,10 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 		},
 		Username: username,
 		Password: password,
+		Transport: &FastHttpTransport{},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)
-
 	if err != nil {
 		return nil, err
 	}
@@ -45,14 +49,31 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 		esIndex = ElasticSearchDefaultIndex
 	}
 
+	// Create the BulkIndexer to batch ES trace submission
+	bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
+		Index:         esIndex,
+		Client:        es,
+		NumWorkers:    esWorkers,
+		FlushBytes:    int(flushBytes),
+		FlushInterval: flushInterval,
+		OnError: func(ctx context.Context, err error) {
+			log.Errorf("Error persisting queries %s", err.Error())
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	return &elasticSearchTransport{
-		cl:      es,
+		cl:	es,
+		bi:	bi,
 		esIndex: esIndex,
 	}, nil
 }
 
 type elasticSearchTransport struct {
 	cl      *elasticsearch.Client
+	bi		esutil.BulkIndexer
 	esIndex string
 }
 
@@ -72,26 +93,18 @@ func (est *elasticSearchTransport) Transport(evt TracerTransportEvent) error {
 		return fmt.Errorf("error while marshaling event: %s", err)
 	}
 
-	req := esapi.IndexRequest{
-		Index:   est.esIndex,
-		Body:    strings.NewReader(string(jsonEvt)),
-		Refresh: "true",
-	}
-
-	// Perform the request with the client.
-	res, err := req.Do(context.Background(), est.cl)
-	if err != nil {
-		return err
-	}
-
-	err = res.Body.Close()
-	if err != nil {
-		return err
-	}
-
-	if res.IsError() {
-		return fmt.Errorf("[%s] Error indexing document ID=%s", res.Status(), req.DocumentID)
-	}
-
-	return nil
+	return est.bi.Add(
+		context.Background(),
+		esutil.BulkIndexerItem{
+			Action: "index",
+			Body:   bytes.NewReader(jsonEvt),
+			OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) {
+				if err != nil {
+					log.Errorf("unable to submit trace - %s", err)
+				} else {
+					log.Errorf("unable to submit trace %s: %s", res.Error.Type, res.Error.Reason)
+				}
+			},
+		},
+	)
 }
diff --git a/node/modules/tracer/fasthttp_transport.go b/node/modules/tracer/fasthttp_transport.go
new file mode 100644
index 000000000..b17bd8132
--- /dev/null
+++ b/node/modules/tracer/fasthttp_transport.go
@@ -0,0 +1,74 @@
+package tracer
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/valyala/fasthttp"
+)
+
+// Transport implements the elastictransport interface with
+// the github.com/valyala/fasthttp HTTP client.
+type FastHttpTransport struct{}
+
+// RoundTrip performs the request and returns a response or error
+func (t *FastHttpTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	freq := fasthttp.AcquireRequest()
+	defer fasthttp.ReleaseRequest(freq)
+
+	fres := fasthttp.AcquireResponse()
+	defer fasthttp.ReleaseResponse(fres)
+
+	t.copyRequest(freq, req)
+
+	err := fasthttp.Do(freq, fres)
+	if err != nil {
+		return nil, err
+	}
+
+	res := &http.Response{Header: make(http.Header)}
+	t.copyResponse(res, fres)
+
+	return res, nil
+}
+
+// copyRequest converts a http.Request to fasthttp.Request
+func (t *FastHttpTransport) copyRequest(dst *fasthttp.Request, src *http.Request) *fasthttp.Request {
+	if src.Method == "GET" && src.Body != nil {
+		src.Method = "POST"
+	}
+
+	dst.SetHost(src.Host)
+	dst.SetRequestURI(src.URL.String())
+
+	dst.Header.SetRequestURI(src.URL.String())
+	dst.Header.SetMethod(src.Method)
+
+	for k, vv := range src.Header {
+		for _, v := range vv {
+			dst.Header.Set(k, v)
+		}
+	}
+
+	if src.Body != nil {
+		dst.SetBodyStream(src.Body, -1)
+	}
+
+	return dst
+}
+
+// copyResponse converts a http.Response to fasthttp.Response
+func (t *FastHttpTransport) copyResponse(dst *http.Response, src *fasthttp.Response) *http.Response {
+	dst.StatusCode = src.StatusCode()
+
+	src.Header.VisitAll(func(k, v []byte) {
+		dst.Header.Set(string(k), string(v))
+	})
+
+	// Cast to a string to make a copy seeing as src.Body() won't
+	// be valid after the response is released back to the pool (fasthttp.ReleaseResponse).
+	dst.Body = ioutil.NopCloser(strings.NewReader(string(src.Body())))
+
+	return dst
+}

From d400d33b1e7c87e36956f24a65010d3f40cd8cc6 Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Wed, 17 May 2023 14:44:19 +0200
Subject: [PATCH 23/46] go format code changes

---
 node/modules/lp2p/pubsub.go               | 8 ++++----
 node/modules/tracer/fasthttp_transport.go | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index 038c129ac..fcee63192 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -381,9 +381,9 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
 	allowTopics = append(allowTopics, drandTopics...)
 	options = append(options,
 		pubsub.WithSubscriptionFilter(
-		pubsub.WrapLimitSubscriptionFilter(
-		pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
-		100)))
+			pubsub.WrapLimitSubscriptionFilter(
+				pubsub.NewAllowlistSubscriptionFilter(allowTopics...),
+				100)))
 
 	var transports []tracer.TracerTransport
 	if in.Cfg.JsonTracer != "" {
@@ -505,7 +505,7 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 
 	case pubsub_pb.TraceEvent_DELIVER_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubDeliverMessage.M(1))
-		
+
 	case pubsub_pb.TraceEvent_REJECT_MESSAGE:
 		stats.Record(context.TODO(), metrics.PubsubRejectMessage.M(1))
 		if trw.traceMessage(evt.GetRejectMessage().GetTopic()) {
diff --git a/node/modules/tracer/fasthttp_transport.go b/node/modules/tracer/fasthttp_transport.go
index b17bd8132..e7e3b8331 100644
--- a/node/modules/tracer/fasthttp_transport.go
+++ b/node/modules/tracer/fasthttp_transport.go
@@ -1,7 +1,7 @@
 package tracer
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 
@@ -68,7 +68,7 @@ func (t *FastHttpTransport) copyResponse(dst *http.Response, src *fasthttp.Respo
 
 	// Cast to a string to make a copy seeing as src.Body() won't
 	// be valid after the response is released back to the pool (fasthttp.ReleaseResponse).
-	dst.Body = ioutil.NopCloser(strings.NewReader(string(src.Body())))
+	dst.Body = io.NopCloser(strings.NewReader(string(src.Body())))
 
 	return dst
 }

From fa8792a65e13b22bcfc3d0d8adf61101ebb59a8a Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Wed, 17 May 2023 14:58:53 +0200
Subject: [PATCH 24/46] make CI happpy with format

---
 node/modules/tracer/elasticsearch_transport.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 3cc50b388..05f9d748b 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -1,10 +1,10 @@
 package tracer
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"bytes"
 	"net/url"
 	"time"
 
@@ -32,8 +32,8 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 		Addresses: []string{
 			conUrl.Scheme + "://" + conUrl.Host,
 		},
-		Username: username,
-		Password: password,
+		Username:  username,
+		Password:  password,
 		Transport: &FastHttpTransport{},
 	}
 
@@ -65,15 +65,15 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 	}
 
 	return &elasticSearchTransport{
-		cl:	es,
-		bi:	bi,
+		cl:      es,
+		bi:      bi,
 		esIndex: esIndex,
 	}, nil
 }
 
 type elasticSearchTransport struct {
 	cl      *elasticsearch.Client
-	bi		esutil.BulkIndexer
+	bi      esutil.BulkIndexer
 	esIndex string
 }
 

From f35fa5757abe60f198aa5658c983411826af18d6 Mon Sep 17 00:00:00 2001
From: gammazero <gammazero@users.noreply.github.com>
Date: Fri, 5 May 2023 12:02:31 -0700
Subject: [PATCH 25/46] Replace use of storetheindex with go-libipni

---
 chain/sub/incoming.go                |  2 +-
 chain/sub/incoming_test.go           |  2 +-
 go.mod                               | 11 +++++------
 go.sum                               | 16 ++++++++--------
 node/modules/storageminer_idxprov.go |  2 +-
 5 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go
index 533314a4f..972773951 100644
--- a/chain/sub/incoming.go
+++ b/chain/sub/incoming.go
@@ -12,7 +12,7 @@ import (
 	bserv "github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/ipni/storetheindex/announce/message"
+	"github.com/ipni/go-libipni/announce/message"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/connmgr"
 	"github.com/libp2p/go-libp2p/core/peer"
diff --git a/chain/sub/incoming_test.go b/chain/sub/incoming_test.go
index 0a9504a88..f54e09049 100644
--- a/chain/sub/incoming_test.go
+++ b/chain/sub/incoming_test.go
@@ -9,7 +9,7 @@ import (
 	"github.com/golang/mock/gomock"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	"github.com/ipni/storetheindex/announce/message"
+	"github.com/ipni/go-libipni/announce/message"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	pb "github.com/libp2p/go-libp2p-pubsub/pb"
 	"github.com/libp2p/go-libp2p/core/peer"
diff --git a/go.mod b/go.mod
index f97f57fc3..fd2fe7b13 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.1
 	github.com/filecoin-project/go-commp-utils v0.1.3
 	github.com/filecoin-project/go-crypto v0.0.1
-	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6
+	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
 	github.com/filecoin-project/go-fil-markets v1.28.1
@@ -87,7 +87,7 @@ require (
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ds-measure v0.2.0
 	github.com/ipfs/go-fs-lock v0.0.7
-	github.com/ipfs/go-graphsync v0.14.5
+	github.com/ipfs/go-graphsync v0.14.6
 	github.com/ipfs/go-ipfs-blockstore v1.3.0
 	github.com/ipfs/go-ipfs-blocksutil v0.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.5
@@ -111,10 +111,9 @@ require (
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
+	github.com/ipni/go-libipni v0.0.8
 	github.com/ipni/index-provider v0.12.0
-	github.com/ipni/storetheindex v0.5.10
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
 	github.com/libp2p/go-libp2p v0.27.3
 	github.com/libp2p/go-libp2p-consensus v0.0.1
@@ -249,7 +248,7 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
-	github.com/ipni/go-libipni v0.0.7 // indirect
+	github.com/ipni/storetheindex v0.5.10 // indirect
 	github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
@@ -286,7 +285,7 @@ require (
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-multicodec v0.8.1 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/nikkolasg/hexjson v0.1.0 // indirect
 	github.com/nkovacs/streamquote v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 9665d04be..cb9040a05 100644
--- a/go.sum
+++ b/go.sum
@@ -312,8 +312,8 @@ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o=
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6 h1:EsbXTWsBKT764qtX4MMJBNXMHoEa+g5Xg01azMqxXX0=
-github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc6/go.mod h1:cX1acvFVWC5EXnnmFPWEFXbO7nLUdSZa+nqgi1QpTpw=
+github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t8tbt1Hz9qENnN4nVr1w99aSWc=
+github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE=
 github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w=
 github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
@@ -738,8 +738,8 @@ github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHL
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
 github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
 github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
-github.com/ipfs/go-graphsync v0.14.5 h1:SKQog4ZABe+yy7OtTsrMoSQfFmEPPi0qP5sl6bFN8xM=
-github.com/ipfs/go-graphsync v0.14.5/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo=
+github.com/ipfs/go-graphsync v0.14.6 h1:NPxvuUy4Z08Mg8dwpBzwgbv/PGLIufSJ1sle6iAX8yo=
+github.com/ipfs/go-graphsync v0.14.6/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
 github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
@@ -885,8 +885,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
-github.com/ipni/go-libipni v0.0.7 h1:L0AnFQagedfJU5mJ7kz0H8P1452brJveOQeS6p3MmbA=
-github.com/ipni/go-libipni v0.0.7/go.mod h1:TlGZaGMGIVpeb6fiwttfY1JgaMnH+HDVPzxgRJJPaQY=
+github.com/ipni/go-libipni v0.0.8 h1:0wLfZRSBG84swmZwmaLKul/iB/FlBkkl9ZcR1ub+Z+w=
+github.com/ipni/go-libipni v0.0.8/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
 github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwSD8jhQ=
 github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4=
 github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
@@ -1327,8 +1327,8 @@ github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6o
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
 github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw=
-github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8=
-github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
+github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
+github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
diff --git a/node/modules/storageminer_idxprov.go b/node/modules/storageminer_idxprov.go
index 3f9a6a989..777c59386 100644
--- a/node/modules/storageminer_idxprov.go
+++ b/node/modules/storageminer_idxprov.go
@@ -64,7 +64,7 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo
 		// If announcements to the network are enabled, then set options for datatransfer publisher.
 		if cfg.Enable {
 			// Join the indexer topic using the market's pubsub instance. Otherwise, the provider
-			// engine would create its own instance of pubsub down the line in go-legs, which has
+			// engine would create its own instance of pubsub down the line in dagsync, which has
 			// no validators by default.
 			t, err := ps.Join(topicName)
 			if err != nil {

From 0a0f540189320aca32a22f1c666ea7d3c6a62d12 Mon Sep 17 00:00:00 2001
From: gammazero <gammazero@users.noreply.github.com>
Date: Wed, 17 May 2023 11:42:34 -0700
Subject: [PATCH 26/46] mod tidy

---
 go.mod | 2 +-
 go.sum | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index fd2fe7b13..4e2da2ca3 100644
--- a/go.mod
+++ b/go.mod
@@ -114,6 +114,7 @@ require (
 	github.com/ipni/go-libipni v0.0.8
 	github.com/ipni/index-provider v0.12.0
 	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
 	github.com/libp2p/go-libp2p v0.27.3
 	github.com/libp2p/go-libp2p-consensus v0.0.1
@@ -248,7 +249,6 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
-	github.com/ipni/storetheindex v0.5.10 // indirect
 	github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
diff --git a/go.sum b/go.sum
index cb9040a05..3db2d5108 100644
--- a/go.sum
+++ b/go.sum
@@ -889,8 +889,6 @@ github.com/ipni/go-libipni v0.0.8 h1:0wLfZRSBG84swmZwmaLKul/iB/FlBkkl9ZcR1ub+Z+w
 github.com/ipni/go-libipni v0.0.8/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
 github.com/ipni/index-provider v0.12.0 h1:R3F6dxxKNv4XkE4GJZNLOG0bDEbBQ/S5iztXwSD8jhQ=
 github.com/ipni/index-provider v0.12.0/go.mod h1:GhyrADJp7n06fqoc1djzkvL4buZYHzV8SoWrlxEo5F4=
-github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
-github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=

From de0726fcb00ca2260736379e877f3c17a59db529 Mon Sep 17 00:00:00 2001
From: Jennifer Wang <jiayingw703@gmail.com>
Date: Wed, 17 May 2023 22:01:04 -0400
Subject: [PATCH 27/46] make ledger sign work

---
 cmd/lotus-shed/verifreg.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go
index 872b4228d..4e2894442 100644
--- a/cmd/lotus-shed/verifreg.go
+++ b/cmd/lotus-shed/verifreg.go
@@ -474,7 +474,7 @@ var verifRegRemoveVerifiedClientDataCapCmd = &cli.Command{
 
 		st, err := multisig.Load(store, vrkState)
 		if err != nil {
-			return err
+			return fmt.Errorf("load vrk failed: %w ", err)
 		}
 
 		signers, err := st.Signers()
@@ -508,14 +508,13 @@ var verifRegRemoveVerifiedClientDataCapCmd = &cli.Command{
 			return err
 		}
 
-		sm, _, err := srv.PublishMessage(ctx, proto, false)
+		sm, err := lcli.InteractiveSend(ctx, cctx, srv, proto)
 		if err != nil {
 			return err
 		}
 
 		msgCid := sm.Cid()
-
-		fmt.Printf("message sent, now waiting on cid: %s\n", msgCid)
+		fmt.Println("sending msg: ", msgCid)
 
 		mwait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
 		if err != nil {

From b08dbde2dad9b7f4fb8a1eb25dd578bea8144a8a Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Thu, 18 May 2023 18:04:10 +0200
Subject: [PATCH 28/46] Update redeclare cmd on Lotus-Workers

Update redeclare cmd on Lotus-Workers
---
 cmd/lotus-worker/storage.go | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/cmd/lotus-worker/storage.go b/cmd/lotus-worker/storage.go
index 9e4bf1c9b..8da35c944 100644
--- a/cmd/lotus-worker/storage.go
+++ b/cmd/lotus-worker/storage.go
@@ -178,6 +178,7 @@ var storageRedeclareCmd = &cli.Command{
 		&cli.BoolFlag{
 			Name:  "drop-missing",
 			Usage: "Drop index entries with missing files",
+			Value: true,
 		},
 	},
 	Action: func(cctx *cli.Context) error {
@@ -188,10 +189,19 @@ var storageRedeclareCmd = &cli.Command{
 		defer closer()
 		ctx := lcli.ReqContext(cctx)
 
+		// check if no argument and no --id or --all flag is provided
+		if cctx.NArg() == 0 && !cctx.IsSet("id") && !cctx.Bool("all") {
+			return xerrors.Errorf("You must specify a storage path, or --id, or --all")
+		}
+
 		if cctx.IsSet("id") && cctx.Bool("all") {
 			return xerrors.Errorf("--id and --all can't be passed at the same time")
 		}
 
+		if cctx.Bool("all") && cctx.NArg() > 0 {
+			return xerrors.Errorf("No additional arguments are expected when --all is set")
+		}
+
 		if cctx.IsSet("id") {
 			id := storiface.ID(cctx.String("id"))
 			return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
@@ -201,6 +211,27 @@ var storageRedeclareCmd = &cli.Command{
 			return nodeApi.StorageRedeclareLocal(ctx, nil, cctx.Bool("drop-missing"))
 		}
 
-		return xerrors.Errorf("either --all or --id must be specified")
+		// As no --id or --all flag is set, we can assume the argument is a path.
+		path := cctx.Args().First()
+		metaFilePath := filepath.Join(path, "sectorstore.json")
+
+		var meta storiface.LocalStorageMeta
+		metaFile, err := os.Open(metaFilePath)
+		if err != nil {
+			return xerrors.Errorf("Failed to open file: %w", err)
+		}
+		defer func() {
+			if closeErr := metaFile.Close(); closeErr != nil {
+				log.Error("Failed to close the file: %v", closeErr)
+			}
+		}()
+
+		err = json.NewDecoder(metaFile).Decode(&meta)
+		if err != nil {
+			return xerrors.Errorf("Failed to decode file: %w", err)
+		}
+
+		id := meta.ID
+		return nodeApi.StorageRedeclareLocal(ctx, &id, cctx.Bool("drop-missing"))
 	},
 }

From e65de697f03be3eccf1d56a65b567095e3fcd2fc Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Thu, 18 May 2023 18:27:53 +0200
Subject: [PATCH 29/46] make docsgen-cli

make docsgen-cli
---
 documentation/en/cli-lotus-worker.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md
index cb3d34729..3ccfadaf6 100644
--- a/documentation/en/cli-lotus-worker.md
+++ b/documentation/en/cli-lotus-worker.md
@@ -148,7 +148,7 @@ USAGE:
 
 OPTIONS:
    --all           redeclare all storage paths (default: false)
-   --drop-missing  Drop index entries with missing files (default: false)
+   --drop-missing  Drop index entries with missing files (default: true)
    --id value      storage path ID
    
 ```

From 3207c695284a4e952d8bc6641969c86320c0c906 Mon Sep 17 00:00:00 2001
From: simlecode <69969590+simlecode@users.noreply.github.com>
Date: Fri, 19 May 2023 16:55:43 +0800
Subject: [PATCH 30/46] opt: cli: If present, print Events Root

---
 cli/state.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cli/state.go b/cli/state.go
index 9031ba870..2750ca680 100644
--- a/cli/state.go
+++ b/cli/state.go
@@ -1528,6 +1528,9 @@ func printMsg(ctx context.Context, api v0api.FullNode, msg cid.Cid, mw *lapi.Msg
 	if err := printReceiptReturn(ctx, api, m, mw.Receipt); err != nil {
 		return err
 	}
+	if mw.Receipt.EventsRoot != nil {
+		fmt.Printf("Events Root: %s\n", mw.Receipt.EventsRoot)
+	}
 
 	return nil
 }

From 40e51baadc6d369d0392d7fc3a83cfe2a0fa3a4c Mon Sep 17 00:00:00 2001
From: Phi <orjan.roren@gmail.com>
Date: Fri, 19 May 2023 14:31:40 +0200
Subject: [PATCH 31/46] Add alerts to `Lotus info` cmd

Add alerts to `Lotus info` cmd
---
 cli/info.go | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/cli/info.go b/cli/info.go
index 007e3655f..8b36be488 100644
--- a/cli/info.go
+++ b/cli/info.go
@@ -20,6 +20,7 @@ import (
 	"github.com/filecoin-project/lotus/api/v1api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/journal/alerting"
 )
 
 var infoCmd = &cli.Command{
@@ -62,6 +63,21 @@ func infoCmdAct(cctx *cli.Context) error {
 	fmt.Printf(" [epoch %s]\n", color.MagentaString(("%d"), status.SyncStatus.Epoch))
 	fmt.Printf("Peers to: [publish messages %d] [publish blocks %d]\n", status.PeerStatus.PeersToPublishMsgs, status.PeerStatus.PeersToPublishBlocks)
 
+	alerts, err := fullapi.LogAlerts(ctx)
+	if err != nil {
+		fmt.Printf("ERROR: getting alerts: %s\n", err)
+	}
+
+	activeAlerts := make([]alerting.Alert, 0)
+	for _, alert := range alerts {
+		if alert.Active {
+			activeAlerts = append(activeAlerts, alert)
+		}
+	}
+	if len(activeAlerts) > 0 {
+		fmt.Printf("%s (check %s)\n", color.RedString("⚠ %d Active alerts", len(activeAlerts)), color.YellowString("lotus log alerts"))
+	}
+
 	//Chain health calculated as percentage: amount of blocks in last finality / very healthy amount of blocks in a finality (900 epochs * 5 blocks per tipset)
 	health := (100 * (900 * status.ChainStatus.BlocksPerTipsetLastFinality) / (900 * 5))
 	switch {

From dfa7fc7723aab1463e2325181579d3899a125dd3 Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 19 May 2023 19:43:56 +0200
Subject: [PATCH 32/46] feat: chainstore: sharded mutex for filling chain
 height index

This PR introduces as sharded mutex within the ChainIndex#GetTipsetByHeight.
It also replaces a go map with xsync.Map which doesn't require locking.

The lock is taken when it appears that ChainIndex filling work should be
started. After claiming the lock, the status of the cache is rechecked,
if the entry is still missing, the fillCache is started.

Thanks to @snissn and @arajasek for debugging and taking initial stabs at this.

Supersedes #10866 and 10885

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/store/index.go                  |  44 ++++---
 go.mod                                |   1 +
 go.sum                                |   2 +
 lib/shardedmutex/shardedmutex.go      |  75 ++++++++++++
 lib/shardedmutex/shardedmutex_test.go | 159 ++++++++++++++++++++++++++
 5 files changed, 266 insertions(+), 15 deletions(-)
 create mode 100644 lib/shardedmutex/shardedmutex.go
 create mode 100644 lib/shardedmutex/shardedmutex_test.go

diff --git a/chain/store/index.go b/chain/store/index.go
index 620cb2dee..5807a2705 100644
--- a/chain/store/index.go
+++ b/chain/store/index.go
@@ -2,18 +2,21 @@ package store
 
 import (
 	"context"
+	"hash/maphash"
 	"os"
 	"strconv"
-	"sync"
 
+	"github.com/puzpuzpuz/xsync/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
 
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/shardedmutex"
 )
 
-var DefaultChainIndexCacheSize = 32 << 15
+// DefaultChainIndexCacheSize no longer sets the maximum size, just the inital size of the map.
+var DefaultChainIndexCacheSize = 1 << 15
 
 func init() {
 	if s := os.Getenv("LOTUS_CHAIN_INDEX_CACHE"); s != "" {
@@ -27,8 +30,9 @@ func init() {
 }
 
 type ChainIndex struct {
-	indexCacheLk sync.Mutex
-	indexCache   map[types.TipSetKey]*lbEntry
+	indexCache *xsync.MapOf[types.TipSetKey, *lbEntry]
+
+	fillCacheLock shardedmutex.ShardedMutexFor[types.TipSetKey]
 
 	loadTipSet loadTipSetFunc
 
@@ -36,11 +40,16 @@ type ChainIndex struct {
 }
 type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error)
 
+func maphashTSK(s maphash.Seed, tsk types.TipSetKey) uint64 {
+	return maphash.Bytes(s, tsk.Bytes())
+}
+
 func NewChainIndex(lts loadTipSetFunc) *ChainIndex {
 	return &ChainIndex{
-		indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize),
-		loadTipSet: lts,
-		skipLength: 20,
+		indexCache:    xsync.NewTypedMapOfPresized[types.TipSetKey, *lbEntry](maphashTSK, DefaultChainIndexCacheSize),
+		fillCacheLock: shardedmutex.NewFor(maphashTSK, 32),
+		loadTipSet:    lts,
+		skipLength:    20,
 	}
 }
 
@@ -59,17 +68,23 @@ func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet,
 		return nil, xerrors.Errorf("failed to round down: %w", err)
 	}
 
-	ci.indexCacheLk.Lock()
-	defer ci.indexCacheLk.Unlock()
 	cur := rounded.Key()
 	for {
-		lbe, ok := ci.indexCache[cur]
+		lbe, ok := ci.indexCache.Load(cur) // check the cache
 		if !ok {
-			fc, err := ci.fillCache(ctx, cur)
-			if err != nil {
-				return nil, xerrors.Errorf("failed to fill cache: %w", err)
+			lk := ci.fillCacheLock.GetLock(cur)
+			lk.Lock()                         // if entry is missing, take the lock
+			lbe, ok = ci.indexCache.Load(cur) // check if someone else added it while we waited for lock
+			if !ok {
+				fc, err := ci.fillCache(ctx, cur)
+				if err != nil {
+					lk.Unlock()
+					return nil, xerrors.Errorf("failed to fill cache: %w", err)
+				}
+				lbe = fc
+				ci.indexCache.Store(cur, lbe)
 			}
-			lbe = fc
+			lk.Unlock()
 		}
 
 		if to == lbe.targetHeight {
@@ -137,7 +152,6 @@ func (ci *ChainIndex) fillCache(ctx context.Context, tsk types.TipSetKey) (*lbEn
 		targetHeight: skipTarget.Height(),
 		target:       skipTarget.Key(),
 	}
-	ci.indexCache[tsk] = lbe
 
 	return lbe, nil
 }
diff --git a/go.mod b/go.mod
index f97f57fc3..bbe737335 100644
--- a/go.mod
+++ b/go.mod
@@ -139,6 +139,7 @@ require (
 	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
 	github.com/polydawn/refmt v0.89.0
 	github.com/prometheus/client_golang v1.14.0
+	github.com/puzpuzpuz/xsync/v2 v2.4.0
 	github.com/raulk/clock v1.1.0
 	github.com/raulk/go-watchdog v1.3.0
 	github.com/stretchr/testify v1.8.2
diff --git a/go.sum b/go.sum
index 9665d04be..84f408724 100644
--- a/go.sum
+++ b/go.sum
@@ -1480,6 +1480,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf33yKI+pyX6T8=
 github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ=
+github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag=
+github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
 github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
diff --git a/lib/shardedmutex/shardedmutex.go b/lib/shardedmutex/shardedmutex.go
new file mode 100644
index 000000000..47b677f51
--- /dev/null
+++ b/lib/shardedmutex/shardedmutex.go
@@ -0,0 +1,75 @@
+package shardedmutex
+
+import (
+	"hash/maphash"
+	"sync"
+)
+
+const cacheline = 64
+
+// padding a mutex to a cacheline improves performance as the cachelines are not contested
+// name     old time/op  new time/op  delta
+// Locks-8  74.6ns ± 7%  12.3ns ± 2%  -83.54%  (p=0.000 n=20+18)
+type paddedMutex struct {
+	mt  sync.Mutex
+	pad [cacheline - 8]uint8
+}
+
+type ShardedMutex struct {
+	shards []paddedMutex
+}
+
+// New creates a new ShardedMutex with N shards
+func New(n_shards int) ShardedMutex {
+	if n_shards < 1 {
+		panic("n_shards cannot be less than 1")
+	}
+	return ShardedMutex{
+		shards: make([]paddedMutex, n_shards),
+	}
+}
+
+func (sm ShardedMutex) Shards() int {
+	return len(sm.shards)
+}
+
+func (sm ShardedMutex) Lock(shard int) {
+	sm.shards[shard].mt.Lock()
+}
+
+func (sm ShardedMutex) Unlock(shard int) {
+	sm.shards[shard].mt.Unlock()
+}
+
+func (sm ShardedMutex) GetLock(shard int) sync.Locker {
+	return &sm.shards[shard].mt
+}
+
+type ShardedMutexFor[K any] struct {
+	inner ShardedMutex
+
+	hasher func(maphash.Seed, K) uint64
+	seed   maphash.Seed
+}
+
+func NewFor[K any](hasher func(maphash.Seed, K) uint64, n_shards int) ShardedMutexFor[K] {
+	return ShardedMutexFor[K]{
+		inner:  New(n_shards),
+		hasher: hasher,
+		seed:   maphash.MakeSeed(),
+	}
+}
+
+func (sm ShardedMutexFor[K]) shardFor(key K) int {
+	return int(sm.hasher(sm.seed, key) % uint64(len(sm.inner.shards)))
+}
+
+func (sm ShardedMutexFor[K]) Lock(key K) {
+	sm.inner.Lock(sm.shardFor(key))
+}
+func (sm ShardedMutexFor[K]) Unlock(key K) {
+	sm.inner.Unlock(sm.shardFor(key))
+}
+func (sm ShardedMutexFor[K]) GetLock(key K) sync.Locker {
+	return sm.inner.GetLock(sm.shardFor(key))
+}
diff --git a/lib/shardedmutex/shardedmutex_test.go b/lib/shardedmutex/shardedmutex_test.go
new file mode 100644
index 000000000..a7d5f7d1d
--- /dev/null
+++ b/lib/shardedmutex/shardedmutex_test.go
@@ -0,0 +1,159 @@
+package shardedmutex
+
+import (
+	"fmt"
+	"hash/maphash"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestLockingDifferentShardsDoesNotBlock(t *testing.T) {
+	shards := 16
+	sm := New(shards)
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-done:
+			return
+		case <-time.After(5 * time.Second):
+			panic("test locked up")
+		}
+	}()
+	for i := 0; i < shards; i++ {
+		sm.Lock(i)
+	}
+
+	close(done)
+}
+func TestLockingSameShardsBlocks(t *testing.T) {
+	shards := 16
+	sm := New(shards)
+	wg := sync.WaitGroup{}
+	wg.Add(shards)
+	ch := make(chan int, shards)
+
+	for i := 0; i < shards; i++ {
+		go func(i int) {
+			if i != 15 {
+				sm.Lock(i)
+			}
+			wg.Done()
+			wg.Wait()
+			sm.Lock((15 + i) % shards)
+			ch <- i
+			sm.Unlock(i)
+		}(i)
+	}
+
+	wg.Wait()
+	for i := 0; i < 2*shards; i++ {
+		runtime.Gosched()
+	}
+	for i := 0; i < shards; i++ {
+		if a := <-ch; a != i {
+			t.Errorf("got %d instead of %d", a, i)
+		}
+	}
+}
+
+func TestShardedByString(t *testing.T) {
+	shards := 16
+	sm := NewFor(maphash.String, shards)
+
+	wg1 := sync.WaitGroup{}
+	wg1.Add(shards * 20)
+	wg2 := sync.WaitGroup{}
+	wg2.Add(shards * 20)
+
+	active := atomic.Int32{}
+	max := atomic.Int32{}
+
+	for i := 0; i < shards*20; i++ {
+		go func(i int) {
+			wg1.Done()
+			wg1.Wait()
+			sm.Lock(fmt.Sprintf("goroutine %d", i))
+			activeNew := active.Add(1)
+			for {
+				curMax := max.Load()
+				if curMax >= activeNew {
+					break
+				}
+				if max.CompareAndSwap(curMax, activeNew) {
+					break
+				}
+			}
+			for j := 0; j < 100; j++ {
+				runtime.Gosched()
+			}
+			active.Add(-1)
+			sm.Unlock(fmt.Sprintf("goroutine %d", i))
+			wg2.Done()
+		}(i)
+	}
+
+	wg2.Wait()
+
+	if max.Load() != 16 {
+		t.Fatal("max load not achieved", max.Load())
+	}
+
+}
+
+func BenchmarkShardedMutex(b *testing.B) {
+	shards := 16
+	sm := New(shards)
+
+	done := atomic.Int32{}
+	go func() {
+		for {
+			sm.Lock(0)
+			sm.Unlock(0)
+			if done.Load() != 0 {
+				return
+			}
+		}
+	}()
+	for i := 0; i < 100; i++ {
+		runtime.Gosched()
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sm.Lock(1)
+		sm.Unlock(1)
+	}
+	done.Add(1)
+}
+
+func BenchmarkShardedMutexOf(b *testing.B) {
+	shards := 16
+	sm := NewFor(maphash.String, shards)
+
+	str1 := "string1"
+	str2 := "string2"
+
+	done := atomic.Int32{}
+	go func() {
+		for {
+			sm.Lock(str1)
+			sm.Unlock(str1)
+			if done.Load() != 0 {
+				return
+			}
+		}
+	}()
+	for i := 0; i < 100; i++ {
+		runtime.Gosched()
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sm.Lock(str2)
+		sm.Unlock(str2)
+	}
+	done.Add(1)
+}

From cad743ec54d4a35dd131b74de37bc4b93051dd3a Mon Sep 17 00:00:00 2001
From: Jakub Sztandera <kubuxu@protocol.ai>
Date: Fri, 19 May 2023 20:02:47 +0200
Subject: [PATCH 33/46] Appease the linter

Signed-off-by: Jakub Sztandera <kubuxu@protocol.ai>
---
 chain/store/index.go             |  2 +-
 lib/shardedmutex/shardedmutex.go | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/chain/store/index.go b/chain/store/index.go
index 5807a2705..8361f4db9 100644
--- a/chain/store/index.go
+++ b/chain/store/index.go
@@ -15,7 +15,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/shardedmutex"
 )
 
-// DefaultChainIndexCacheSize no longer sets the maximum size, just the inital size of the map.
+// DefaultChainIndexCacheSize no longer sets the maximum size, just the initial size of the map.
 var DefaultChainIndexCacheSize = 1 << 15
 
 func init() {
diff --git a/lib/shardedmutex/shardedmutex.go b/lib/shardedmutex/shardedmutex.go
index 47b677f51..922ac3994 100644
--- a/lib/shardedmutex/shardedmutex.go
+++ b/lib/shardedmutex/shardedmutex.go
@@ -20,12 +20,12 @@ type ShardedMutex struct {
 }
 
 // New creates a new ShardedMutex with N shards
-func New(n_shards int) ShardedMutex {
-	if n_shards < 1 {
+func New(nShards int) ShardedMutex {
+	if nShards < 1 {
 		panic("n_shards cannot be less than 1")
 	}
 	return ShardedMutex{
-		shards: make([]paddedMutex, n_shards),
+		shards: make([]paddedMutex, nShards),
 	}
 }
 
@@ -52,9 +52,9 @@ type ShardedMutexFor[K any] struct {
 	seed   maphash.Seed
 }
 
-func NewFor[K any](hasher func(maphash.Seed, K) uint64, n_shards int) ShardedMutexFor[K] {
+func NewFor[K any](hasher func(maphash.Seed, K) uint64, nShards int) ShardedMutexFor[K] {
 	return ShardedMutexFor[K]{
-		inner:  New(n_shards),
+		inner:  New(nShards),
 		hasher: hasher,
 		seed:   maphash.MakeSeed(),
 	}

From bb92984dcad19f1d9cd22a4eab3d7340ebc40f83 Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Mon, 22 May 2023 09:13:10 +0200
Subject: [PATCH 34/46] tracer: simplify the valid Iwant logic

---
 node/modules/lp2p/pubsub.go | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index fcee63192..d0381c5dc 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -609,10 +609,7 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 			}
 		}
 		// check if we have any of iwant msgs (it doesn't classify per topic - just msg.ID)
-		var validIwant bool = false
-		if len(iwant) > 0 {
-			validIwant = true
-		}
+		validIwant := len(iwant) > 0
 
 		// trace the event if any of the flags was triggered
 		if validIhave || validIwant || validTopic {
@@ -648,10 +645,8 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 			}
 		}
 		// check if there was any of the Iwant msgs
-		var validIwant bool = false
-		if len(iwant) > 0 {
-			validIwant = true
-		}
+		validIwant := len(iwant) > 0 
+
 		// trace the msgs if any of the flags was triggered
 		if validIhave || validIwant || validTopic {
 			if trw.lp2pTracer != nil {

From 42d2f4d7e48104c4b8c6f19720e4eef369976442 Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Mon, 22 May 2023 09:43:07 +0200
Subject: [PATCH 35/46] tracer: move es transport to net/http + format

---
 node/modules/lp2p/pubsub.go                   |  2 +-
 .../modules/tracer/elasticsearch_transport.go |  3 +-
 node/modules/tracer/fasthttp_transport.go     | 74 -------------------
 3 files changed, 3 insertions(+), 76 deletions(-)
 delete mode 100644 node/modules/tracer/fasthttp_transport.go

diff --git a/node/modules/lp2p/pubsub.go b/node/modules/lp2p/pubsub.go
index d0381c5dc..33a03f844 100644
--- a/node/modules/lp2p/pubsub.go
+++ b/node/modules/lp2p/pubsub.go
@@ -645,7 +645,7 @@ func (trw *tracerWrapper) Trace(evt *pubsub_pb.TraceEvent) {
 			}
 		}
 		// check if there was any of the Iwant msgs
-		validIwant := len(iwant) > 0 
+		validIwant := len(iwant) > 0
 
 		// trace the msgs if any of the flags was triggered
 		if validIhave || validIwant || validTopic {
diff --git a/node/modules/tracer/elasticsearch_transport.go b/node/modules/tracer/elasticsearch_transport.go
index 05f9d748b..e54e0eba2 100644
--- a/node/modules/tracer/elasticsearch_transport.go
+++ b/node/modules/tracer/elasticsearch_transport.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"time"
 
@@ -34,7 +35,7 @@ func NewElasticSearchTransport(connectionString string, elasticsearchIndex strin
 		},
 		Username:  username,
 		Password:  password,
-		Transport: &FastHttpTransport{},
+		Transport: &http.Transport{},
 	}
 
 	es, err := elasticsearch.NewClient(cfg)
diff --git a/node/modules/tracer/fasthttp_transport.go b/node/modules/tracer/fasthttp_transport.go
deleted file mode 100644
index e7e3b8331..000000000
--- a/node/modules/tracer/fasthttp_transport.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package tracer
-
-import (
-	"io"
-	"net/http"
-	"strings"
-
-	"github.com/valyala/fasthttp"
-)
-
-// Transport implements the elastictransport interface with
-// the github.com/valyala/fasthttp HTTP client.
-type FastHttpTransport struct{}
-
-// RoundTrip performs the request and returns a response or error
-func (t *FastHttpTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	freq := fasthttp.AcquireRequest()
-	defer fasthttp.ReleaseRequest(freq)
-
-	fres := fasthttp.AcquireResponse()
-	defer fasthttp.ReleaseResponse(fres)
-
-	t.copyRequest(freq, req)
-
-	err := fasthttp.Do(freq, fres)
-	if err != nil {
-		return nil, err
-	}
-
-	res := &http.Response{Header: make(http.Header)}
-	t.copyResponse(res, fres)
-
-	return res, nil
-}
-
-// copyRequest converts a http.Request to fasthttp.Request
-func (t *FastHttpTransport) copyRequest(dst *fasthttp.Request, src *http.Request) *fasthttp.Request {
-	if src.Method == "GET" && src.Body != nil {
-		src.Method = "POST"
-	}
-
-	dst.SetHost(src.Host)
-	dst.SetRequestURI(src.URL.String())
-
-	dst.Header.SetRequestURI(src.URL.String())
-	dst.Header.SetMethod(src.Method)
-
-	for k, vv := range src.Header {
-		for _, v := range vv {
-			dst.Header.Set(k, v)
-		}
-	}
-
-	if src.Body != nil {
-		dst.SetBodyStream(src.Body, -1)
-	}
-
-	return dst
-}
-
-// copyResponse converts a http.Response to fasthttp.Response
-func (t *FastHttpTransport) copyResponse(dst *http.Response, src *fasthttp.Response) *http.Response {
-	dst.StatusCode = src.StatusCode()
-
-	src.Header.VisitAll(func(k, v []byte) {
-		dst.Header.Set(string(k), string(v))
-	})
-
-	// Cast to a string to make a copy seeing as src.Body() won't
-	// be valid after the response is released back to the pool (fasthttp.ReleaseResponse).
-	dst.Body = io.NopCloser(strings.NewReader(string(src.Body())))
-
-	return dst
-}

From d9a65223abab80f32c474841f8daa75db7ce0a36 Mon Sep 17 00:00:00 2001
From: cortze <cortze@protonmail.com>
Date: Mon, 22 May 2023 11:59:39 +0200
Subject: [PATCH 36/46] tidy dependencies

---
 go.mod | 2 --
 go.sum | 4 ----
 2 files changed, 6 deletions(-)

diff --git a/go.mod b/go.mod
index 1b091e095..f97f57fc3 100644
--- a/go.mod
+++ b/go.mod
@@ -144,7 +144,6 @@ require (
 	github.com/stretchr/testify v1.8.2
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/urfave/cli/v2 v2.16.3
-	github.com/valyala/fasthttp v1.45.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
@@ -180,7 +179,6 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akavel/rsrc v0.8.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
diff --git a/go.sum b/go.sum
index 4b4dd2a9f..9665d04be 100644
--- a/go.sum
+++ b/go.sum
@@ -93,8 +93,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
-github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
-github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -1636,8 +1634,6 @@ github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk=
 github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.45.0 h1:zPkkzpIn8tdHZUrVa6PzYd0i5verqiPSkgTd3bSUcpA=
-github.com/valyala/fasthttp v1.45.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=

From fff151f07ec7c04f807d706005d8a7448aa215b7 Mon Sep 17 00:00:00 2001
From: Maciej Witowski <maciej.witowski@protocol.ai>
Date: Mon, 22 May 2023 17:16:30 +0200
Subject: [PATCH 37/46] Fix button color so it doesn't look disabled when
 hovering over

---
 cmd/lotus-fountain/site/main.css | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-fountain/site/main.css b/cmd/lotus-fountain/site/main.css
index e2670d123..a1d3070c2 100644
--- a/cmd/lotus-fountain/site/main.css
+++ b/cmd/lotus-fountain/site/main.css
@@ -66,7 +66,7 @@ button {
 }
 
 button:hover {
-    background-color: #555;
+    background-color: #4c7eff;
 }
 
 a:link {

From c233b4317d608f6a103475aea836eda9db59eb02 Mon Sep 17 00:00:00 2001
From: Maciej Witowski <maciej.witowski@protocol.ai>
Date: Mon, 22 May 2023 17:33:10 +0200
Subject: [PATCH 38/46] Small UI improvements for faucet

---
 cmd/lotus-fountain/site/datacap.html | 5 +++--
 cmd/lotus-fountain/site/funds.html   | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/cmd/lotus-fountain/site/datacap.html b/cmd/lotus-fountain/site/datacap.html
index f42a3a55f..f99f0532a 100644
--- a/cmd/lotus-fountain/site/datacap.html
+++ b/cmd/lotus-fountain/site/datacap.html
@@ -15,12 +15,13 @@
 <div class="Index">
     <div class="Index-nodes">
         <div class="Index-node">
-            GRANT DATACAP
+            <h3>Grant datacap</h3>
+            <p>Please input your address to receive a data cap on the Calibration Testnet.</p>
         </div>
         <div class="Index-node">
             <form action='/datacap' method='post' id='datacap-form'>
                 <span>Enter destination address:</span>
-                <input type='text' name='address' style="width: 300px">
+                <input type='text' name='address' style="width: 300px" placeholder="t0/1/2/3/4 or 0xETH">
                 <button class="g-recaptcha"
                         data-sitekey="{{ . }}"
                         data-callback='onSubmit'
diff --git a/cmd/lotus-fountain/site/funds.html b/cmd/lotus-fountain/site/funds.html
index ae7a4041b..a09d73964 100644
--- a/cmd/lotus-fountain/site/funds.html
+++ b/cmd/lotus-fountain/site/funds.html
@@ -15,12 +15,13 @@
 <div class="Index">
     <div class="Index-nodes">
         <div class="Index-node">
-            SENDING FUNDS
+            <h3>Send funds</h3>
+            <p>Please input your address to receive test FIL (tFIL) on the Calibration Testnet. This faucet dispenses 100 tFIL.</p>
         </div>
         <div class="Index-node">
             <form action='/send' method='post' id='funds-form'>
                 <span>Enter destination address:</span>
-				<input type='text' name='address' style="width: 300px">
+                <input type='text' name='address' style="width: 300px" placeholder="Enter t0/1/2/3/4 or 0xETH">
 				<button class="g-recaptcha" 
 						data-sitekey="{{ . }}"
 						data-callback='onSubmit' 

From 00e34436b3eb18b901c67657345cec420cf02433 Mon Sep 17 00:00:00 2001
From: Yolan Romailler <yolan@protocol.ai>
Date: Mon, 22 May 2023 21:45:24 +0200
Subject: [PATCH 39/46] Addressing code review comments

Signed-off-by: Yolan Romailler <AnomalRoil@users.noreply.github.com>
---
 chain/types/blockmsg_test.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/chain/types/blockmsg_test.go b/chain/types/blockmsg_test.go
index 20cbe324d..02a622768 100644
--- a/chain/types/blockmsg_test.go
+++ b/chain/types/blockmsg_test.go
@@ -16,7 +16,7 @@ func TestDecodeBlockMsg(t *testing.T) {
 		want    *BlockMsg
 		wantErr bool
 	}{
-		{"decode empty BlockMsg with extra data at the end", []byte{0x83, 0xf6, 0x80, 0x80, 0x20}, new(BlockMsg), true},
+		{"decode empty BlockMsg with extra data at the end", []byte{0x83, 0xf6, 0x80, 0x80, 0x20}, nil, true},
 		{"decode valid empty BlockMsg", []byte{0x83, 0xf6, 0x80, 0x80}, new(BlockMsg), false},
 		{"decode invalid cbor", []byte{0x83, 0xf6, 0x80}, nil, true},
 	}
@@ -27,14 +27,14 @@ func TestDecodeBlockMsg(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := DecodeBlockMsg(data)
 			if wantErr {
-				assert.Errorf(t, err, "DecodeBlockMsg(%v)", data)
+				assert.Errorf(t, err, "DecodeBlockMsg(%x)", data)
 				return
 			}
-			assert.NoErrorf(t, err, "DecodeBlockMsg(%v)", data)
-			assert.Equalf(t, want, got, "DecodeBlockMsg(%v)", data)
+			assert.NoErrorf(t, err, "DecodeBlockMsg(%x)", data)
+			assert.Equalf(t, want, got, "DecodeBlockMsg(%x)", data)
 			serialized, err := got.Serialize()
-			assert.NoErrorf(t, err, "DecodeBlockMsg(%v)", data)
-			assert.Equalf(t, serialized, data, "DecodeBlockMsg(%v)", data)
+			assert.NoErrorf(t, err, "DecodeBlockMsg(%x)", data)
+			assert.Equalf(t, serialized, data, "DecodeBlockMsg(%x)", data)
 		})
 	}
 }

From 6ec413b7adb88fcdf426d74997fa5590122dad60 Mon Sep 17 00:00:00 2001
From: simlecode <69969590+simlecode@users.noreply.github.com>
Date: Tue, 23 May 2023 17:43:39 +0800
Subject: [PATCH 40/46] fix: not call RUnlock

---
 chain/messagepool/check.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/chain/messagepool/check.go b/chain/messagepool/check.go
index 17829057a..fdec910c4 100644
--- a/chain/messagepool/check.go
+++ b/chain/messagepool/check.go
@@ -35,6 +35,7 @@ func (mp *MessagePool) CheckPendingMessages(ctx context.Context, from address.Ad
 	mp.lk.RLock()
 	mset, ok, err := mp.getPendingMset(ctx, from)
 	if err != nil {
+		mp.lk.RUnlock()
 		return nil, xerrors.Errorf("errored while getting pending mset: %w", err)
 	}
 	if ok {
@@ -70,6 +71,7 @@ func (mp *MessagePool) CheckReplaceMessages(ctx context.Context, replace []*type
 			msgMap[m.From] = mmap
 			mset, ok, err := mp.getPendingMset(ctx, m.From)
 			if err != nil {
+				mp.lk.RUnlock()
 				return nil, xerrors.Errorf("errored while getting pending mset: %w", err)
 			}
 			if ok {
@@ -153,6 +155,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 			mp.lk.RLock()
 			mset, ok, err := mp.getPendingMset(ctx, m.From)
 			if err != nil {
+				mp.lk.RUnlock()
 				return nil, xerrors.Errorf("errored while getting pending mset: %w", err)
 			}
 			if ok && !interned {

From 41203f7b8e602124aea4d2f9c23d18bee8f8e68e Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Wed, 24 May 2023 21:06:03 -0700
Subject: [PATCH 41/46] pass the right g-recaptcha data

---
 cmd/lotus-fountain/site/datacap.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-fountain/site/datacap.html b/cmd/lotus-fountain/site/datacap.html
index f99f0532a..7434b8e48 100644
--- a/cmd/lotus-fountain/site/datacap.html
+++ b/cmd/lotus-fountain/site/datacap.html
@@ -6,7 +6,7 @@
 	<script src="https://www.google.com/recaptcha/api.js"></script>
 	<script>
    		function onSubmit(token) {
-     	document.getElementById("funds-form").submit();
+     	document.getElementById("datacap-form").submit();
    	}
 	</script>
 

From 479018e3b3b8b330549ddcb1794e590dfd134f12 Mon Sep 17 00:00:00 2001
From: Jianhui Xie <Jesse.Xie@protocol.ai>
Date: Wed, 24 May 2023 22:09:29 -0700
Subject: [PATCH 42/46] set default data-cap same as MinVerifiedDealSize

---
 cmd/lotus-fountain/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go
index 2e61fcb6d..f6d503c2f 100644
--- a/cmd/lotus-fountain/main.go
+++ b/cmd/lotus-fountain/main.go
@@ -78,7 +78,7 @@ var runCmd = &cli.Command{
 		&cli.Uint64Flag{
 			Name:    "data-cap",
 			EnvVars: []string{"LOTUS_DATACAP_AMOUNT"},
-			Value:   10240,
+			Value:   verifregtypes9.MinVerifiedDealSize.Uint64(),
 		},
 		&cli.Float64Flag{
 			Name:  "captcha-threshold",

From 7af42796dbbfe8d1e42f638bf68eb791a1e4aa79 Mon Sep 17 00:00:00 2001
From: PARTHIBAN <parthibanb224@gmail.com>
Date: Fri, 26 May 2023 20:12:10 +0530
Subject: [PATCH 43/46] Update gas_balancing.md

correct the sentences
---
 documentation/misc/gas_balancing.md | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/documentation/misc/gas_balancing.md b/documentation/misc/gas_balancing.md
index 213c9681e..8252cd46d 100644
--- a/documentation/misc/gas_balancing.md
+++ b/documentation/misc/gas_balancing.md
@@ -2,8 +2,8 @@
 
 The gas balancing process targets to set gas costs of syscalls to be in line with
 10 gas per nanosecond on reference hardware.
-The process can be either performed for all syscalls based on existing messages and chain or targeted
-at single syscall.
+The process can be either performed for all syscalls based on existing messages and chains or targeted
+at a single syscall.
 
 #### Reference hardware
 
@@ -12,14 +12,14 @@ may be subject to change.
 
 ### Complete gas balancing
 
-Complete gas balancing is performed using `lotus-bench` the process is based on importing a chain export
+Complete gas balancing is performed using a `lotus-bench` the process is based on importing a chain export
 and collecting gas traces which are later aggregated.
 
 Before building `lotus-bench` make sure `EnableDetailedTracing` in `chain/vm/runtime.go` is set to `true`.
 
 The process can be started using `./lotus-bench import` with `--car` flag set to the location of
-CAR chain export. `--start-epoch` and `--end-epoch` can be used to to limit the range of epochs to run
-the benchmark. Note that state tree of `start-epoch` needs to be in the CAR file or has to be previously computed
+CAR chain export. `--start-epoch` and `--end-epoch` can be used to limit the range of epochs to run
+the benchmark. Note that the state tree of `start-epoch` needs to be in the CAR file or has to be previously computed
 to work.
 
 The output will be a `bench.json` file containing information about every syscall invoked
@@ -29,7 +29,7 @@ spare space.
 After the bench run is complete the `bench.json` file can be analyzed with `./lotus-bench import analyze bench.json`.
 
 It will compute means, standard deviations and co-variances (when applicable) of syscall runtimes. 
-The output is in nanoseconds, so the gas values for syscalls should be 10x that. In cases where co-variance of
+The output is in nanoseconds, so the gas values for syscalls should be 10x that. In cases where the co-variance of
 execution time to some parameter is evaluated, the strength of the correlation should be taken into account.
 
 #### Special cases
@@ -40,15 +40,15 @@ during block execution (when gas traces are formed) objects are only written to
 
 ### Targeted gas balancing
 
-In some cases complete gas balancing is infeasible, either new syscall gets introduced or
+In some cases complete gas balancing is infeasible, either a new syscall gets introduced or
 complete balancing is too time consuming.
 
-In these cases the recommended way to estimate gas for given syscall is to perform an `in-vivo` benchmark.
+In these cases, the recommended way to estimate gas for a given syscall is to perform an `in-vivo` benchmark.
 In the past `in-vitro` as in standalone benchmarks were found to be highly inaccurate when compared to results
 of real execution.
 
-A in-vivo benchmark can be performed by running an example of such syscall during block execution.
-The best place to hook-in such benchmark is message execution loop in
-`chain/stmgr/stmgr.go` in `ApplyBlocks()`. Depending of time required to complete the syscall it might be
+An in-vivo benchmark can be performed by running an example of such a syscall during block execution.
+The best place to hook-in such a benchmark is the message execution loop in
+`chain/stmgr/stmgr.go` in `ApplyBlocks()`. Depending on the time required to complete the syscall it might be
 advisable to run the execution only once every few messages.
 

From a4c94957e6d5361c673e7f4effcbcf2c7c9b4fb3 Mon Sep 17 00:00:00 2001
From: Fridrik Asmundsson <fridrik01@gmail.com>
Date: Sat, 27 May 2023 10:10:30 +0000
Subject: [PATCH 44/46] Update message when user hits CRTL+C to not imply
 handling SIGSTOP but SIGTERM/SIGKILL

---
 cmd/lotus-shed/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go
index 6aebb264f..a9e06f7c0 100644
--- a/cmd/lotus-shed/main.go
+++ b/cmd/lotus-shed/main.go
@@ -127,7 +127,7 @@ func main() {
 	go func() {
 		<-c
 		cancel()
-		fmt.Println("Received interrupt, stopping... Press CTRL+C again to force stop")
+		fmt.Println("Received interrupt, shutting down... Press CTRL+C again to force shutdown")
 		<-c
 		fmt.Println("Forcing stop")
 		os.Exit(1)

From c0942ebc2c831675a92522124c6c319c7fd98311 Mon Sep 17 00:00:00 2001
From: Aayush <arajasek94@gmail.com>
Date: Wed, 20 Jul 2022 14:11:01 -0400
Subject: [PATCH 45/46] feat: cli: Make compact partitions cmd better

---
 cmd/lotus-miner/sectors.go | 133 ++++++++++++++++++++++++++++---------
 1 file changed, 100 insertions(+), 33 deletions(-)

diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go
index 4d241b282..968e92174 100644
--- a/cmd/lotus-miner/sectors.go
+++ b/cmd/lotus-miner/sectors.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -2056,7 +2057,6 @@ func yesno(b bool) string {
 	return color.RedString("NO")
 }
 
-// TODO simulate this call if --really-do-it is not used
 var sectorsCompactPartitionsCmd = &cli.Command{
 	Name:  "compact-partitions",
 	Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible",
@@ -2082,12 +2082,7 @@ var sectorsCompactPartitionsCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		if !cctx.Bool("really-do-it") {
-			fmt.Println("Pass --really-do-it to actually execute this action")
-			return nil
-		}
-
-		api, acloser, err := lcli.GetFullNodeAPI(cctx)
+		fullNodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
 		}
@@ -2100,7 +2095,7 @@ var sectorsCompactPartitionsCmd = &cli.Command{
 			return err
 		}
 
-		minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		minfo, err := fullNodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
 		if err != nil {
 			return err
 		}
@@ -2116,46 +2111,118 @@ var sectorsCompactPartitionsCmd = &cli.Command{
 		}
 		fmt.Printf("compacting %d paritions\n", len(parts))
 
+		var makeMsgForPartitions func(partitionsBf bitfield.BitField) ([]*types.Message, error)
+		makeMsgForPartitions = func(partitionsBf bitfield.BitField) ([]*types.Message, error) {
+			params := miner.CompactPartitionsParams{
+				Deadline:   deadline,
+				Partitions: partitionsBf,
+			}
+
+			sp, aerr := actors.SerializeParams(&params)
+			if aerr != nil {
+				return nil, xerrors.Errorf("serializing params: %w", err)
+			}
+
+			msg := &types.Message{
+				From:   minfo.Worker,
+				To:     maddr,
+				Method: builtin.MethodsMiner.CompactPartitions,
+				Value:  big.Zero(),
+				Params: sp,
+			}
+
+			estimatedMsg, err := fullNodeAPI.GasEstimateMessageGas(ctx, msg, nil, types.EmptyTSK)
+			if err != nil && xerrors.Is(err, &api.ErrOutOfGas{}) {
+				// the message is too big -- split into 2
+				partitionsSlice, err := partitionsBf.All(math.MaxUint64)
+				if err != nil {
+					return nil, err
+				}
+
+				partitions1 := bitfield.New()
+				for i := 0; i < len(partitionsSlice)/2; i++ {
+					partitions1.Set(uint64(i))
+				}
+
+				msgs1, err := makeMsgForPartitions(partitions1)
+				if err != nil {
+					return nil, err
+				}
+
+				// time for the second half
+				partitions2 := bitfield.New()
+				for i := len(partitionsSlice) / 2; i < len(partitionsSlice); i++ {
+					partitions2.Set(uint64(i))
+				}
+
+				msgs2, err := makeMsgForPartitions(partitions2)
+				if err != nil {
+					return nil, err
+				}
+
+				return append(msgs1, msgs2...), nil
+			} else if err != nil {
+				return nil, err
+			}
+
+			return []*types.Message{estimatedMsg}, nil
+		}
+
 		partitions := bitfield.New()
 		for _, partition := range parts {
 			partitions.Set(uint64(partition))
 		}
 
-		params := miner.CompactPartitionsParams{
-			Deadline:   deadline,
-			Partitions: partitions,
-		}
-
-		sp, err := actors.SerializeParams(&params)
+		msgs, err := makeMsgForPartitions(partitions)
 		if err != nil {
-			return xerrors.Errorf("serializing params: %w", err)
+			return xerrors.Errorf("failed to make messages: %w", err)
 		}
 
-		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
-			From:   minfo.Worker,
-			To:     maddr,
-			Method: builtin.MethodsMiner.CompactPartitions,
-			Value:  big.Zero(),
-			Params: sp,
-		}, nil)
-		if err != nil {
-			return xerrors.Errorf("mpool push: %w", err)
+		// Actually send the messages if really-do-it provided, simulate otherwise
+		if cctx.Bool("really-do-it") {
+			smsgs, err := fullNodeAPI.MpoolBatchPushMessage(ctx, msgs, nil)
+			if err != nil {
+				return xerrors.Errorf("mpool push: %w", err)
+			}
+
+			if len(smsgs) == 1 {
+				fmt.Printf("Requested compact partitions in message %s\n", smsgs[0].Cid())
+			} else {
+				fmt.Printf("Requested compact partitions in %d messages\n\n", len(smsgs))
+				for _, v := range smsgs {
+					fmt.Println(v.Cid())
+				}
+			}
+
+			for _, v := range smsgs {
+				wait, err := fullNodeAPI.StateWaitMsg(ctx, v.Cid(), 2)
+				if err != nil {
+					return err
+				}
+
+				// check it executed successfully
+				if wait.Receipt.ExitCode.IsError() {
+					fmt.Println(cctx.App.Writer, "compact partitions msg %s failed!", v.Cid())
+					return err
+				}
+			}
+
+			return nil
 		}
 
-		fmt.Printf("Requested compact partitions in message %s\n", smsg.Cid())
+		for i, v := range msgs {
+			fmt.Printf("total of %d CompactPartitions msgs would be sent\n", len(msgs))
 
-		wait, err := api.StateWaitMsg(ctx, smsg.Cid(), 0)
-		if err != nil {
-			return err
-		}
+			estMsg, err := fullNodeAPI.GasEstimateMessageGas(ctx, v, nil, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
 
-		// check it executed successfully
-		if wait.Receipt.ExitCode.IsError() {
-			fmt.Println(cctx.App.Writer, "compact partitions failed!")
-			return err
+			fmt.Printf("msg %d would cost up to %s\n", i+1, types.FIL(estMsg.RequiredFunds()))
 		}
 
 		return nil
+
 	},
 }
 

From f5daf9526193e319a3015b820fdd13f33997ab6c Mon Sep 17 00:00:00 2001
From: Jennifer Wang <jiayingw703@gmail.com>
Date: Wed, 17 May 2023 11:08:35 -0400
Subject: [PATCH 46/46] update interop

---
 build/params_interop.go | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/build/params_interop.go b/build/params_interop.go
index 2d8d366e9..04fc777f5 100644
--- a/build/params_interop.go
+++ b/build/params_interop.go
@@ -49,16 +49,11 @@ var UpgradeHyperdriveHeight = abi.ChainEpoch(-16)
 var UpgradeChocolateHeight = abi.ChainEpoch(-17)
 var UpgradeOhSnapHeight = abi.ChainEpoch(-18)
 var UpgradeSkyrHeight = abi.ChainEpoch(-19)
+var UpgradeSharkHeight = abi.ChainEpoch(-20)
+var UpgradeHyggeHeight = abi.ChainEpoch(-21)
+var UpgradeLightningHeight = abi.ChainEpoch(-22)
 
-const UpgradeSharkHeight = abi.ChainEpoch(-20)
-
-const UpgradeHyggeHeight = abi.ChainEpoch(100)
-
-// ??????????
-const UpgradeLightningHeight = 200
-
-// ??????????????????
-const UpgradeThunderHeight = 300
+const UpgradeThunderHeight = 50
 
 var DrandSchedule = map[abi.ChainEpoch]DrandEnum{
 	0: DrandMainnet,