use msgindex as epoch=>tipsetkey index to speed up eth_getBlockByNumber queries

This commit updates the GetMsgInfo to include the execution tipset of
the message (if it exists). This allows us to call GetTipSetByCid
instead of GetTipsetByHeight which should be considerably faster and
help speed up both StateSearcMessage and StateWaitForMessage calls.

.circleci/config.yml

@@ -818,6 +818,12 @@ workflows:
             - build
           suite: itest-mpool_push_with_uuid
           target: "./itests/mpool_push_with_uuid_test.go"
+      - test:
+          name: test-itest-msgindex
+          requires:
+            - build
+          suite: itest-msgindex
+          target: "./itests/msgindex_test.go"
       - test:
           name: test-itest-multisig
           requires:

Dockerfile

@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.18.8-buster AS lotus-builder
+FROM golang:1.19.7-buster AS lotus-builder
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

Dockerfile.lotus

@@ -1,6 +1,6 @@
 ##### DEPRECATED
 
-FROM golang:1.18.8-buster AS builder-deps
+FROM golang:1.19.7-buster AS builder-deps
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

README.md

@@ -71,10 +71,10 @@ For other distributions you can find the required dependencies [here.](https://l
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.18.8 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.19.7 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.18.8.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.19.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**

api/api_full.go

@@ -297,8 +297,10 @@ type FullNode interface {
 	MpoolGetNonce(context.Context, address.Address) (uint64, error) //perm:read
 	MpoolSub(context.Context) (<-chan MpoolUpdate, error)           //perm:read
 
-	// MpoolClear clears pending messages from the mpool
-	MpoolClear(context.Context, bool) error //perm:write
+	// MpoolClear clears pending messages from the mpool.
+	// If clearLocal is true, ALL messages will be cleared.
+	// If clearLocal is false, local messages will be protected, all others will be cleared.
+	MpoolClear(ctx context.Context, clearLocal bool) error //perm:write
 
 	// MpoolGetConfig returns (a copy of) the current mpool config
 	MpoolGetConfig(context.Context) (*types.MpoolConfig, error) //perm:read

api/api_gateway.go

@@ -33,6 +33,9 @@ import (
 //  * Generate openrpc blobs
 
 type Gateway interface {
+	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error)
+	GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
+	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*InvocResult, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)

api/proxy_gen.go

@@ -726,6 +726,8 @@ type GatewayMethods struct {
 
 	EthUnsubscribe func(p0 context.Context, p1 ethtypes.EthSubscriptionID) (bool, error) ``
 
+	GasEstimateGasPremium func(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) ``
+
 	GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
 
 	MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) ``
@@ -768,12 +770,16 @@ type GatewayMethods struct {
 
 	StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``
 
+	StateMinerSectorCount func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MinerSectors, error) ``
+
 	StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) ``
 
 	StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) ``
 
 	StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) ``
 
+	StateReplay func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*InvocResult, error) ``
+
 	StateSearchMsg func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) ``
 
 	StateSectorGetInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorOnChainInfo, error) ``
@@ -4614,6 +4620,17 @@ func (s *GatewayStub) EthUnsubscribe(p0 context.Context, p1 ethtypes.EthSubscrip
 	return false, ErrNotSupported
 }
 
+func (s *GatewayStruct) GasEstimateGasPremium(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) {
+	if s.Internal.GasEstimateGasPremium == nil {
+		return *new(types.BigInt), ErrNotSupported
+	}
+	return s.Internal.GasEstimateGasPremium(p0, p1, p2, p3, p4)
+}
+
+func (s *GatewayStub) GasEstimateGasPremium(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) {
+	return *new(types.BigInt), ErrNotSupported
+}
+
 func (s *GatewayStruct) GasEstimateMessageGas(p0 context.Context, p1 *types.Message, p2 *MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) {
 	if s.Internal.GasEstimateMessageGas == nil {
 		return nil, ErrNotSupported
@@ -4845,6 +4862,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) StateMinerSectorCount(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MinerSectors, error) {
+	if s.Internal.StateMinerSectorCount == nil {
+		return *new(MinerSectors), ErrNotSupported
+	}
+	return s.Internal.StateMinerSectorCount(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateMinerSectorCount(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (MinerSectors, error) {
+	return *new(MinerSectors), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
 	if s.Internal.StateNetworkName == nil {
 		return *new(dtypes.NetworkName), ErrNotSupported
@@ -4878,6 +4906,17 @@ func (s *GatewayStub) StateReadState(p0 context.Context, p1 address.Address, p2
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) StateReplay(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*InvocResult, error) {
+	if s.Internal.StateReplay == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateReplay(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateReplay(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*InvocResult, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *GatewayStruct) StateSearchMsg(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) {
 	if s.Internal.StateSearchMsg == nil {
 		return nil, ErrNotSupported

api/v0api/gateway.go

@@ -35,6 +35,9 @@ import (
 //  * Generate openrpc blobs
 
 type Gateway interface {
+	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)
+	GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
+	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainPutObj(context.Context, blocks.Block) error
 	ChainHead(ctx context.Context) (*types.TipSet, error)

api/v0api/proxy_gen.go

@@ -449,6 +449,8 @@ type GatewayMethods struct {
 
 	ChainReadObj func(p0 context.Context, p1 cid.Cid) ([]byte, error) ``
 
+	GasEstimateGasPremium func(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) ``
+
 	GasEstimateMessageGas func(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) ``
 
 	MpoolGetNonce func(p0 context.Context, p1 address.Address) (uint64, error) ``
@@ -487,10 +489,14 @@ type GatewayMethods struct {
 
 	StateMinerProvingDeadline func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*dline.Info, error) ``
 
+	StateMinerSectorCount func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerSectors, error) ``
+
 	StateNetworkName func(p0 context.Context) (dtypes.NetworkName, error) ``
 
 	StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (abinetwork.Version, error) ``
 
+	StateReplay func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*api.InvocResult, error) ``
+
 	StateSearchMsg func(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) ``
 
 	StateSectorGetInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorOnChainInfo, error) ``
@@ -2674,6 +2680,17 @@ func (s *GatewayStub) ChainReadObj(p0 context.Context, p1 cid.Cid) ([]byte, erro
 	return *new([]byte), ErrNotSupported
 }
 
+func (s *GatewayStruct) GasEstimateGasPremium(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) {
+	if s.Internal.GasEstimateGasPremium == nil {
+		return *new(types.BigInt), ErrNotSupported
+	}
+	return s.Internal.GasEstimateGasPremium(p0, p1, p2, p3, p4)
+}
+
+func (s *GatewayStub) GasEstimateGasPremium(p0 context.Context, p1 uint64, p2 address.Address, p3 int64, p4 types.TipSetKey) (types.BigInt, error) {
+	return *new(types.BigInt), ErrNotSupported
+}
+
 func (s *GatewayStruct) GasEstimateMessageGas(p0 context.Context, p1 *types.Message, p2 *api.MessageSendSpec, p3 types.TipSetKey) (*types.Message, error) {
 	if s.Internal.GasEstimateMessageGas == nil {
 		return nil, ErrNotSupported
@@ -2883,6 +2900,17 @@ func (s *GatewayStub) StateMinerProvingDeadline(p0 context.Context, p1 address.A
 	return nil, ErrNotSupported
 }
 
+func (s *GatewayStruct) StateMinerSectorCount(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerSectors, error) {
+	if s.Internal.StateMinerSectorCount == nil {
+		return *new(api.MinerSectors), ErrNotSupported
+	}
+	return s.Internal.StateMinerSectorCount(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateMinerSectorCount(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (api.MinerSectors, error) {
+	return *new(api.MinerSectors), ErrNotSupported
+}
+
 func (s *GatewayStruct) StateNetworkName(p0 context.Context) (dtypes.NetworkName, error) {
 	if s.Internal.StateNetworkName == nil {
 		return *new(dtypes.NetworkName), ErrNotSupported
@@ -2905,6 +2933,17 @@ func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey
 	return *new(abinetwork.Version), ErrNotSupported
 }
 
+func (s *GatewayStruct) StateReplay(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*api.InvocResult, error) {
+	if s.Internal.StateReplay == nil {
+		return nil, ErrNotSupported
+	}
+	return s.Internal.StateReplay(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateReplay(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid) (*api.InvocResult, error) {
+	return nil, ErrNotSupported
+}
+
 func (s *GatewayStruct) StateSearchMsg(p0 context.Context, p1 cid.Cid) (*api.MsgLookup, error) {
 	if s.Internal.StateSearchMsg == nil {
 		return nil, ErrNotSupported

blockstore/badger/blockstore.go

@@ -396,6 +396,9 @@ func (b *Blockstore) doCopy(from, to *badger.DB) error {
 	if workers < 2 {
 		workers = 2
 	}
+	if workers > 8 {
+		workers = 8
+	}
 
 	stream := from.NewStream()
 	stream.NumGo = workers

blockstore/ipfs.go

@@ -3,7 +3,7 @@ package blockstore
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
+	"io"
 
 	"github.com/ipfs/go-cid"
 	httpapi "github.com/ipfs/go-ipfs-http-client"
@@ -103,7 +103,7 @@ func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, er
 		return nil, xerrors.Errorf("getting ipfs block: %w", err)
 	}
 
-	data, err := ioutil.ReadAll(rd)
+	data, err := io.ReadAll(rd)
 	if err != nil {
 		return nil, err
 	}

build/params_2k.go

@@ -6,6 +6,7 @@ package build
 import (
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -139,3 +140,7 @@ const BootstrapPeerThreshold = 1
 const Eip155ChainId = 31415926
 
 var WhitelistedBlock = cid.Undef
+
+// Reducing the delivery delay for equivocation of
+// consistent broadcast to just half a second.
+var CBDeliveryDelay = 500 * time.Millisecond

build/params_butterfly.go

@@ -4,6 +4,8 @@
 package build
 
 import (
+	"time"
+
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/go-address"
@@ -87,3 +89,8 @@ const BootstrapPeerThreshold = 2
 const Eip155ChainId = 3141592
 
 var WhitelistedBlock = cid.Undef
+
+// CBDeliveryDelay is the delay before deliver in the synchronous consistent broadcast.
+// This determines the wait time for the detection of potential equivocations.
+// It is a variable instead of a constant so it can be conveniently configured in tests
+var CBDeliveryDelay = 2 * time.Second

build/params_calibnet.go

@@ -6,6 +6,7 @@ package build
 import (
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -122,3 +123,8 @@ const BootstrapPeerThreshold = 4
 const Eip155ChainId = 314159
 
 var WhitelistedBlock = cid.Undef
+
+// CBDeliveryDelay is the delay before deliver in the synchronous consistent broadcast.
+// This determines the wait time for the detection of potential equivocations.
+// It is a variable instead of a constant so it can be conveniently configured in tests
+var CBDeliveryDelay = 2 * time.Second

build/params_interop.go

@@ -6,6 +6,7 @@ package build
 import (
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -128,3 +129,8 @@ const BootstrapPeerThreshold = 2
 const Eip155ChainId = 3141592
 
 var WhitelistedBlock = cid.Undef
+
+// CBDeliveryDelay is the delay before deliver in the synchronous consistent broadcast.
+// This determines the wait time for the detection of potential equivocations.
+// It is a variable instead of a constant so it can be conveniently configured in tests
+var CBDeliveryDelay = 2 * time.Second

build/params_mainnet.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -137,3 +138,8 @@ const Eip155ChainId = 314
 
 // we skip checks on message validity in this block to sidestep the zero-bls signature
 var WhitelistedBlock = MustParseCid("bafy2bzaceapyg2uyzk7vueh3xccxkuwbz3nxewjyguoxvhx77malc2lzn2ybi")
+
+// CBDeliveryDelay is the delay before deliver in the synchronous consistent broadcast.
+// This determines the wait time for the detection of potential equivocations.
+// It is a variable instead of a constant so it can be conveniently configured in tests
+var CBDeliveryDelay = 2 * time.Second

build/params_testground.go

@@ -9,6 +9,7 @@ package build
 
 import (
 	"math/big"
+	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -135,3 +136,7 @@ const BootstrapPeerThreshold = 1
 // ChainId defines the chain ID used in the Ethereum JSON-RPC endpoint.
 // As per https://github.com/ethereum-lists/chains
 const Eip155ChainId = 31415926
+
+// Reducing the delivery delay for equivocation of
+// consistent broadcast to just half a second.
+var CBDeliveryDelay = 500 * time.Millisecond

chain/actors/agen/main.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"go/format"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -66,7 +65,7 @@ func generateAdapters() error {
 		}
 
 		{
-			af, err := ioutil.ReadFile(filepath.Join(actDir, "actor.go.template"))
+			af, err := os.ReadFile(filepath.Join(actDir, "actor.go.template"))
 			if err != nil {
 				return xerrors.Errorf("loading actor template: %w", err)
 			}
@@ -90,7 +89,7 @@ func generateAdapters() error {
 				return err
 			}
 
-			if err := ioutil.WriteFile(filepath.Join(actDir, fmt.Sprintf("%s.go", act)), fmted, 0666); err != nil {
+			if err := os.WriteFile(filepath.Join(actDir, fmt.Sprintf("%s.go", act)), fmted, 0666); err != nil {
 				return err
 			}
 		}
@@ -100,7 +99,7 @@ func generateAdapters() error {
 }
 
 func generateState(actDir string, versions []int) error {
-	af, err := ioutil.ReadFile(filepath.Join(actDir, "state.go.template"))
+	af, err := os.ReadFile(filepath.Join(actDir, "state.go.template"))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil // skip
@@ -123,7 +122,7 @@ func generateState(actDir string, versions []int) error {
 			return err
 		}
 
-		if err := ioutil.WriteFile(filepath.Join(actDir, fmt.Sprintf("v%d.go", version)), b.Bytes(), 0666); err != nil {
+		if err := os.WriteFile(filepath.Join(actDir, fmt.Sprintf("v%d.go", version)), b.Bytes(), 0666); err != nil {
 			return err
 		}
 	}
@@ -132,7 +131,7 @@ func generateState(actDir string, versions []int) error {
 }
 
 func generateMessages(actDir string) error {
-	af, err := ioutil.ReadFile(filepath.Join(actDir, "message.go.template"))
+	af, err := os.ReadFile(filepath.Join(actDir, "message.go.template"))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil // skip
@@ -155,7 +154,7 @@ func generateMessages(actDir string) error {
 			return err
 		}
 
-		if err := ioutil.WriteFile(filepath.Join(actDir, fmt.Sprintf("message%d.go", version)), b.Bytes(), 0666); err != nil {
+		if err := os.WriteFile(filepath.Join(actDir, fmt.Sprintf("message%d.go", version)), b.Bytes(), 0666); err != nil {
 			return err
 		}
 	}
@@ -165,7 +164,7 @@ func generateMessages(actDir string) error {
 
 func generatePolicy(policyPath string) error {
 
-	pf, err := ioutil.ReadFile(policyPath + ".template")
+	pf, err := os.ReadFile(policyPath + ".template")
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil // skip
@@ -187,7 +186,7 @@ func generatePolicy(policyPath string) error {
 		return err
 	}
 
-	if err := ioutil.WriteFile(policyPath, b.Bytes(), 0666); err != nil {
+	if err := os.WriteFile(policyPath, b.Bytes(), 0666); err != nil {
 		return err
 	}
 
@@ -196,7 +195,7 @@ func generatePolicy(policyPath string) error {
 
 func generateBuiltin(builtinPath string) error {
 
-	bf, err := ioutil.ReadFile(builtinPath + ".template")
+	bf, err := os.ReadFile(builtinPath + ".template")
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil // skip
@@ -218,7 +217,7 @@ func generateBuiltin(builtinPath string) error {
 		return err
 	}
 
-	if err := ioutil.WriteFile(builtinPath, b.Bytes(), 0666); err != nil {
+	if err := os.WriteFile(builtinPath, b.Bytes(), 0666); err != nil {
 		return err
 	}
 
@@ -227,7 +226,7 @@ func generateBuiltin(builtinPath string) error {
 
 func generateRegistry(registryPath string) error {
 
-	bf, err := ioutil.ReadFile(registryPath + ".template")
+	bf, err := os.ReadFile(registryPath + ".template")
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil // skip
@@ -248,7 +247,7 @@ func generateRegistry(registryPath string) error {
 		return err
 	}
 
-	if err := ioutil.WriteFile(registryPath, b.Bytes(), 0666); err != nil {
+	if err := os.WriteFile(registryPath, b.Bytes(), 0666); err != nil {
 		return err
 	}
 

chain/consensus/compute_state.go

@@ -109,6 +109,7 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context,
 			TipSetGetter:   stmgr.TipSetGetterForTipset(sm.ChainStore(), ts),
 			Tracing:        vmTracing,
 			ReturnEvents:   sm.ChainStore().IsStoringEvents(),
+			ExecutionLane:  vm.ExecutionLanePriority,
 		}
 
 		return sm.VMConstructor()(ctx, vmopt)

chain/gen/gen.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"os"
 	"sync/atomic"
 	"time"
 
@@ -34,6 +34,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/rand"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -167,7 +168,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
 
 	maddr1 := genesis2.MinerAddress(0)
 
-	m1temp, err := ioutil.TempDir("", "preseal")
+	m1temp, err := os.MkdirTemp("", "preseal")
 	if err != nil {
 		return nil, err
 	}
@@ -179,7 +180,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
 
 	maddr2 := genesis2.MinerAddress(1)
 
-	m2temp, err := ioutil.TempDir("", "preseal")
+	m2temp, err := os.MkdirTemp("", "preseal")
 	if err != nil {
 		return nil, err
 	}
@@ -256,7 +257,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS
 	//return nil, xerrors.Errorf("creating drand beacon: %w", err)
 	//}
 
-	sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac, ds)
+	sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), sys, us, beac, ds, index.DummyMsgIndex)
 	if err != nil {
 		return nil, xerrors.Errorf("initing stmgr: %w", err)
 	}

chain/index/interface.go

@@ -0,0 +1,66 @@
+package index
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+var ErrNotFound = errors.New("message not found")
+var ErrClosed = errors.New("index closed")
+
+// MsgInfo is the Message metadata the index tracks.
+type MsgInfo struct {
+	// the message this record refers to
+	Message cid.Cid
+	// the tipset where this message was included
+	TipSet cid.Cid
+	// the epoch where this message was included
+	Epoch abi.ChainEpoch
+}
+
+// MsgIndex is the interface to the message index
+type MsgIndex interface {
+	// GetMsgInfo looks up the message in index and retrieves its metadata and execution
+	// tipset cid.
+	// The lookup is done using the onchain message Cid; that is the signed message Cid
+	// for SECP messages and unsigned message Cid for BLS messages.
+	GetMsgInfo(ctx context.Context, mCid cid.Cid) (MsgInfo, cid.Cid, error)
+	// Close closes the index
+	Close() error
+}
+
+// TipsetIndex is the interface to the tipset index
+type TipsetIndex interface {
+	// GetTipsetCID returns the tipset cid for the given epoch
+	GetTipsetCID(ctx context.Context, epoch abi.ChainEpoch) (*cid.Cid, error)
+	// Close closes the index
+	Close() error
+}
+
+type dummyMsgIndex struct{}
+
+func (dummyMsgIndex) GetMsgInfo(ctx context.Context, mCid cid.Cid) (MsgInfo, cid.Cid, error) {
+	return MsgInfo{}, cid.Undef, ErrNotFound
+}
+
+func (dummyMsgIndex) Close() error {
+	return nil
+}
+
+var DummyMsgIndex MsgIndex = dummyMsgIndex{}
+
+type dummyTipsetIndex struct{}
+
+func (dummyTipsetIndex) GetTipsetCID(ctx context.Context, epoch abi.ChainEpoch) (*cid.Cid, error) {
+	return nil, ErrNotFound
+}
+
+func (dummyTipsetIndex) Close() error {
+	return nil
+}
+
+var DummyTipsetIndex TipsetIndex = dummyTipsetIndex{}

chain/index/msgindex.go

@@ -0,0 +1,604 @@
+package index
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"io/fs"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
+	_ "github.com/mattn/go-sqlite3"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+var log = logging.Logger("msgindex")
+
+var dbName = "msgindex.db"
+var dbDefs = []string{
+	`CREATE TABLE IF NOT EXISTS messages (
+     cid VARCHAR(80) PRIMARY KEY ON CONFLICT REPLACE,
+     tipset_cid VARCHAR(80) NOT NULL,
+     epoch INTEGER NOT NULL
+   )`,
+	`CREATE INDEX IF NOT EXISTS tipset_cids ON messages (tipset_cid)`,
+	`CREATE INDEX IF NOT EXISTS tipset_epochs ON messages (epoch)`,
+	`CREATE TABLE IF NOT EXISTS _meta (
+    	version UINT64 NOT NULL UNIQUE
+	)`,
+	`INSERT OR IGNORE INTO _meta (version) VALUES (1)`,
+}
+var dbPragmas = []string{}
+
+const (
+	// prepared stmts
+	dbqGetMessageInfo       = "SELECT tipset_cid, epoch FROM messages WHERE cid = ?"
+	dbqInsertMessage        = "INSERT INTO messages VALUES (?, ?, ?)"
+	dbqDeleteTipsetMessages = "DELETE FROM messages WHERE tipset_cid = ?"
+	dbqGetTipsetByEpoch     = "SELECT tipset_cid FROM messages WHERE epoch = ? LIMIT 1"
+	// reconciliation
+	dbqCountMessages         = "SELECT COUNT(*) FROM messages"
+	dbqMinEpoch              = "SELECT MIN(epoch) FROM messages"
+	dbqCountTipsetMessages   = "SELECT COUNT(*) FROM messages WHERE tipset_cid = ?"
+	dbqDeleteMessagesByEpoch = "DELETE FROM messages WHERE epoch >= ?"
+)
+
+// coalescer configuration (TODO: use observer instead)
+// these are exposed to make tests snappy
+var (
+	CoalesceMinDelay      = time.Second
+	CoalesceMaxDelay      = 15 * time.Second
+	CoalesceMergeInterval = time.Second
+)
+
+// chain store interface; we could use store.ChainStore directly,
+// but this simplifies unit testing.
+type ChainStore interface {
+	SubscribeHeadChanges(f store.ReorgNotifee)
+	MessagesForTipset(ctx context.Context, ts *types.TipSet) ([]types.ChainMsg, error)
+	GetHeaviestTipSet() *types.TipSet
+	GetTipSetFromKey(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
+}
+
+var _ ChainStore = (*store.ChainStore)(nil)
+
+type msgIndex struct {
+	cs ChainStore
+
+	db               *sql.DB
+	selectMsgStmt    *sql.Stmt
+	selectTipsetStmt *sql.Stmt
+	insertMsgStmt    *sql.Stmt
+	deleteTipSetStmt *sql.Stmt
+
+	sema chan struct{}
+	mx   sync.Mutex
+	pend []headChange
+
+	cancel  func()
+	workers sync.WaitGroup
+	closeLk sync.RWMutex
+	closed  bool
+}
+
+var _ MsgIndex = (*msgIndex)(nil)
+
+type headChange struct {
+	rev []*types.TipSet
+	app []*types.TipSet
+}
+
+func NewMsgIndex(lctx context.Context, basePath string, cs ChainStore) (MsgIndex, error) {
+	var (
+		dbPath string
+		exists bool
+		err    error
+	)
+
+	err = os.MkdirAll(basePath, 0755)
+	if err != nil {
+		return nil, xerrors.Errorf("error creating msgindex base directory: %w", err)
+	}
+
+	dbPath = path.Join(basePath, dbName)
+	_, err = os.Stat(dbPath)
+	switch {
+	case err == nil:
+		exists = true
+
+	case errors.Is(err, fs.ErrNotExist):
+
+	case err != nil:
+		return nil, xerrors.Errorf("error stating msgindex database: %w", err)
+	}
+
+	db, err := sql.Open("sqlite3", dbPath)
+	if err != nil {
+		// TODO [nice to have]: automaticaly delete corrupt databases
+		//      but for now we can just error and let the operator delete.
+		return nil, xerrors.Errorf("error opening msgindex database: %w", err)
+	}
+
+	if err := prepareDB(db); err != nil {
+		return nil, xerrors.Errorf("error creating msgindex database: %w", err)
+	}
+
+	// TODO we may consider populating the index when first creating the db
+	if exists {
+		if err := reconcileIndex(db, cs); err != nil {
+			return nil, xerrors.Errorf("error reconciling msgindex database: %w", err)
+		}
+	}
+
+	ctx, cancel := context.WithCancel(lctx)
+
+	msgIndex := &msgIndex{
+		db:     db,
+		cs:     cs,
+		sema:   make(chan struct{}, 1),
+		cancel: cancel,
+	}
+
+	err = msgIndex.prepareStatements()
+	if err != nil {
+		if err := db.Close(); err != nil {
+			log.Errorf("error closing msgindex database: %s", err)
+		}
+
+		return nil, xerrors.Errorf("error preparing msgindex database statements: %w", err)
+	}
+
+	rnf := store.WrapHeadChangeCoalescer(
+		msgIndex.onHeadChange,
+		CoalesceMinDelay,
+		CoalesceMaxDelay,
+		CoalesceMergeInterval,
+	)
+	cs.SubscribeHeadChanges(rnf)
+
+	msgIndex.workers.Add(1)
+	go msgIndex.background(ctx)
+
+	return msgIndex, nil
+}
+
+func PopulateAfterSnapshot(lctx context.Context, basePath string, cs ChainStore) error {
+	err := os.MkdirAll(basePath, 0755)
+	if err != nil {
+		return xerrors.Errorf("error creating msgindex base directory: %w", err)
+	}
+
+	dbPath := path.Join(basePath, dbName)
+
+	// if a database already exists, we try to delete it and create a new one
+	if _, err := os.Stat(dbPath); err == nil {
+		if err = os.Remove(dbPath); err != nil {
+			return xerrors.Errorf("msgindex already exists at %s and can't be deleted", dbPath)
+		}
+	}
+
+	db, err := sql.Open("sqlite3", dbPath)
+	if err != nil {
+		return xerrors.Errorf("error opening msgindex database: %w", err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Errorf("error closing msgindex database: %s", err)
+		}
+	}()
+
+	if err := prepareDB(db); err != nil {
+		return xerrors.Errorf("error creating msgindex database: %w", err)
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return xerrors.Errorf("error when starting transaction: %w", err)
+	}
+
+	rollback := func() {
+		if err := tx.Rollback(); err != nil {
+			log.Errorf("error in rollback: %s", err)
+		}
+	}
+
+	insertStmt, err := tx.Prepare(dbqInsertMessage)
+	if err != nil {
+		rollback()
+		return xerrors.Errorf("error preparing insertStmt: %w", err)
+	}
+
+	curTs := cs.GetHeaviestTipSet()
+	startHeight := curTs.Height()
+	for curTs != nil {
+		tscid, err := curTs.Key().Cid()
+		if err != nil {
+			rollback()
+			return xerrors.Errorf("error computing tipset cid: %w", err)
+		}
+
+		tskey := tscid.String()
+		epoch := int64(curTs.Height())
+
+		msgs, err := cs.MessagesForTipset(lctx, curTs)
+		if err != nil {
+			log.Infof("stopping import after %d tipsets", startHeight-curTs.Height())
+			break
+		}
+
+		for _, msg := range msgs {
+			key := msg.Cid().String()
+			if _, err := insertStmt.Exec(key, tskey, epoch); err != nil {
+				rollback()
+				return xerrors.Errorf("error inserting message: %w", err)
+			}
+		}
+
+		curTs, err = cs.GetTipSetFromKey(lctx, curTs.Parents())
+		if err != nil {
+			rollback()
+			return xerrors.Errorf("error walking chain: %w", err)
+		}
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return xerrors.Errorf("error committing transaction: %w", err)
+	}
+
+	return nil
+}
+
+// init utilities
+func prepareDB(db *sql.DB) error {
+	for _, stmt := range dbDefs {
+		if _, err := db.Exec(stmt); err != nil {
+			return xerrors.Errorf("error executing sql statement '%s': %w", stmt, err)
+		}
+	}
+
+	for _, stmt := range dbPragmas {
+		if _, err := db.Exec(stmt); err != nil {
+			return xerrors.Errorf("error executing sql statement '%s': %w", stmt, err)
+		}
+	}
+
+	return nil
+}
+
+func reconcileIndex(db *sql.DB, cs ChainStore) error {
+	// Invariant: after reconciliation, every tipset in the index is in the current chain; ie either
+	//  the chain head or reachable by walking the chain.
+	// Algorithm:
+	//  1. Count messages in index; if none, trivially reconciled.
+	//     TODO we may consider populating the index in that case
+	//  2. Find the minimum tipset in the index; this will mark the end of the reconciliation walk
+	//  3. Walk from current tipset until we find a tipset in the index.
+	//  4. Delete (revert!) all tipsets above the found tipset.
+	//  5. If the walk ends in the boundary epoch, then delete everything.
+	//
+
+	row := db.QueryRow(dbqCountMessages)
+
+	var result int64
+	if err := row.Scan(&result); err != nil {
+		return xerrors.Errorf("error counting messages: %w", err)
+	}
+
+	if result == 0 {
+		return nil
+	}
+
+	row = db.QueryRow(dbqMinEpoch)
+	if err := row.Scan(&result); err != nil {
+		return xerrors.Errorf("error finding boundary epoch: %w", err)
+	}
+
+	boundaryEpoch := abi.ChainEpoch(result)
+
+	countMsgsStmt, err := db.Prepare(dbqCountTipsetMessages)
+	if err != nil {
+		return xerrors.Errorf("error preparing statement: %w", err)
+	}
+
+	curTs := cs.GetHeaviestTipSet()
+	for curTs != nil && curTs.Height() >= boundaryEpoch {
+		tsCid, err := curTs.Key().Cid()
+		if err != nil {
+			return xerrors.Errorf("error computing tipset cid: %w", err)
+		}
+
+		key := tsCid.String()
+		row = countMsgsStmt.QueryRow(key)
+		if err := row.Scan(&result); err != nil {
+			return xerrors.Errorf("error counting messages: %w", err)
+		}
+
+		if result > 0 {
+			// found it!
+			boundaryEpoch = curTs.Height() + 1
+			break
+		}
+
+		// walk up
+		parents := curTs.Parents()
+		curTs, err = cs.GetTipSetFromKey(context.TODO(), parents)
+		if err != nil {
+			return xerrors.Errorf("error walking chain: %w", err)
+		}
+	}
+
+	// delete everything above the minEpoch
+	if _, err = db.Exec(dbqDeleteMessagesByEpoch, int64(boundaryEpoch)); err != nil {
+		return xerrors.Errorf("error deleting stale reorged out message: %w", err)
+	}
+
+	return nil
+}
+
+func (x *msgIndex) prepareStatements() error {
+	stmt, err := x.db.Prepare(dbqGetMessageInfo)
+	if err != nil {
+		return xerrors.Errorf("prepare selectMsgStmt: %w", err)
+	}
+	x.selectMsgStmt = stmt
+
+	stmt, err = x.db.Prepare(dbqGetTipsetByEpoch)
+	if err != nil {
+		return xerrors.Errorf("prepare selectTipsetStmt: %w", err)
+	}
+	x.selectTipsetStmt = stmt
+
+	stmt, err = x.db.Prepare(dbqInsertMessage)
+	if err != nil {
+		return xerrors.Errorf("prepare insertMsgStmt: %w", err)
+	}
+	x.insertMsgStmt = stmt
+
+	stmt, err = x.db.Prepare(dbqDeleteTipsetMessages)
+	if err != nil {
+		return xerrors.Errorf("prepare deleteTipSetStmt: %w", err)
+	}
+	x.deleteTipSetStmt = stmt
+
+	return nil
+}
+
+// head change notifee
+func (x *msgIndex) onHeadChange(rev, app []*types.TipSet) error {
+	x.closeLk.RLock()
+	defer x.closeLk.RUnlock()
+
+	if x.closed {
+		return nil
+	}
+
+	// do it in the background to avoid blocking head change processing
+	x.mx.Lock()
+	x.pend = append(x.pend, headChange{rev: rev, app: app})
+	pendLen := len(x.pend)
+	x.mx.Unlock()
+
+	// complain loudly if this is building backlog
+	if pendLen > 10 {
+		log.Warnf("message index head change processing is building backlog: %d pending head changes", pendLen)
+	}
+
+	select {
+	case x.sema <- struct{}{}:
+	default:
+	}
+
+	return nil
+}
+
+func (x *msgIndex) background(ctx context.Context) {
+	defer x.workers.Done()
+
+	for {
+		select {
+		case <-x.sema:
+			err := x.processHeadChanges(ctx)
+			if err != nil {
+				// we can't rely on an inconsistent index, so shut it down.
+				log.Errorf("error processing head change notifications: %s; shutting down message index", err)
+				if err2 := x.Close(); err2 != nil {
+					log.Errorf("error shutting down index: %s", err2)
+				}
+			}
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (x *msgIndex) processHeadChanges(ctx context.Context) error {
+	x.mx.Lock()
+	pend := x.pend
+	x.pend = nil
+	x.mx.Unlock()
+
+	tx, err := x.db.Begin()
+	if err != nil {
+		return xerrors.Errorf("error creating transaction: %w", err)
+	}
+
+	for _, hc := range pend {
+		for _, ts := range hc.rev {
+			if err := x.doRevert(ctx, tx, ts); err != nil {
+				if err2 := tx.Rollback(); err2 != nil {
+					log.Errorf("error rolling back transaction: %s", err2)
+				}
+				return xerrors.Errorf("error reverting %s: %w", ts, err)
+			}
+		}
+
+		for _, ts := range hc.app {
+			if err := x.doApply(ctx, tx, ts); err != nil {
+				if err2 := tx.Rollback(); err2 != nil {
+					log.Errorf("error rolling back transaction: %s", err2)
+				}
+				return xerrors.Errorf("error applying %s: %w", ts, err)
+			}
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (x *msgIndex) doRevert(ctx context.Context, tx *sql.Tx, ts *types.TipSet) error {
+	tskey, err := ts.Key().Cid()
+	if err != nil {
+		return xerrors.Errorf("error computing tipset cid: %w", err)
+	}
+
+	key := tskey.String()
+	_, err = tx.Stmt(x.deleteTipSetStmt).Exec(key)
+	return err
+}
+
+func (x *msgIndex) doApply(ctx context.Context, tx *sql.Tx, ts *types.TipSet) error {
+	tscid, err := ts.Key().Cid()
+	if err != nil {
+		return xerrors.Errorf("error computing tipset cid: %w", err)
+	}
+
+	tskey := tscid.String()
+	epoch := int64(ts.Height())
+
+	msgs, err := x.cs.MessagesForTipset(ctx, ts)
+	if err != nil {
+		return xerrors.Errorf("error retrieving messages for tipset %s: %w", ts, err)
+	}
+
+	insertStmt := tx.Stmt(x.insertMsgStmt)
+	for _, msg := range msgs {
+		key := msg.Cid().String()
+		if _, err := insertStmt.Exec(key, tskey, epoch); err != nil {
+			return xerrors.Errorf("error inserting message: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// interface
+func (x *msgIndex) GetMsgInfo(ctx context.Context, mCid cid.Cid) (MsgInfo, cid.Cid, error) {
+	x.closeLk.RLock()
+	defer x.closeLk.RUnlock()
+
+	if x.closed {
+		return MsgInfo{}, cid.Undef, ErrClosed
+	}
+
+	// fetch message from index (if it exists)
+	//
+	var tipset string
+	var epoch int64
+	err := x.selectMsgStmt.QueryRow(mCid.String()).Scan(&tipset, &epoch)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			// mCid not in index, its fine
+			return MsgInfo{}, cid.Undef, ErrNotFound
+		}
+
+		return MsgInfo{}, cid.Undef, xerrors.Errorf("error querying msgindex database: %w", err)
+	}
+	tsCid, err := cid.Decode(tipset)
+	if err != nil {
+		return MsgInfo{}, cid.Undef, xerrors.Errorf("error decoding tipset cid: %w", err)
+	}
+
+	msgInfo := MsgInfo{
+		Message: mCid,
+		TipSet:  tsCid,
+		Epoch:   abi.ChainEpoch(epoch),
+	}
+
+	// fetch execution tipset of message (if it exists)
+	//
+	var xTipset string
+	err = x.selectTipsetStmt.QueryRow(epoch + 1).Scan(&xTipset)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			// execution tipset not in index, its fine
+			return msgInfo, cid.Undef, nil
+		}
+
+		return MsgInfo{}, cid.Undef, xerrors.Errorf("error querying for execution tipset: %w", err)
+	}
+	xtsCid, err := cid.Decode(xTipset)
+	if err != nil {
+		return MsgInfo{}, cid.Undef, xerrors.Errorf("error decoding execution tipset cid: %w", err)
+	}
+
+	return msgInfo, xtsCid, nil
+}
+
+func (x *msgIndex) GetTipsetCID(ctx context.Context, epoch abi.ChainEpoch) (*cid.Cid, error) {
+	x.closeLk.RLock()
+	defer x.closeLk.RUnlock()
+
+	if x.closed {
+		return nil, ErrClosed
+	}
+
+	var tipset string
+
+	row := x.selectTipsetStmt.QueryRow(epoch)
+	err := row.Scan(&tipset)
+	switch {
+	case err == sql.ErrNoRows:
+		return nil, ErrNotFound
+
+	case err != nil:
+		return nil, xerrors.Errorf("error querying msgindex database: %w", err)
+	}
+
+	tipsetCid, err := cid.Decode(tipset)
+	if err != nil {
+		return nil, xerrors.Errorf("error decoding tipset cid: %w", err)
+	}
+	return &tipsetCid, nil
+}
+
+func (x *msgIndex) Close() error {
+	x.closeLk.Lock()
+	defer x.closeLk.Unlock()
+
+	if x.closed {
+		return nil
+	}
+
+	x.closed = true
+
+	x.cancel()
+	x.workers.Wait()
+
+	return x.db.Close()
+}
+
+// informal apis for itests; not exposed in the main interface
+func (x *msgIndex) CountMessages() (int64, error) {
+	x.closeLk.RLock()
+	defer x.closeLk.RUnlock()
+
+	if x.closed {
+		return 0, ErrClosed
+	}
+
+	var result int64
+	row := x.db.QueryRow(dbqCountMessages)
+	err := row.Scan(&result)
+	return result, err
+}

chain/index/msgindex_test.go

@@ -0,0 +1,298 @@
+package index
+
+import (
+	"context"
+	"errors"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-libipfs/blocks"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/types/mock"
+)
+
+func TestBasicMsgIndex(t *testing.T) {
+	// the most basic of tests:
+	// 1. Create an index with mock chain store
+	// 2. Advance the chain for a few tipsets
+	// 3. Verify that the index contains all messages with the correct tipset/epoch
+	cs := newMockChainStore()
+	cs.genesis()
+
+	tmp := t.TempDir()
+	t.Cleanup(func() { _ = os.RemoveAll(tmp) })
+
+	msgIndex, err := NewMsgIndex(context.Background(), tmp, cs)
+	require.NoError(t, err)
+
+	defer msgIndex.Close() //nolint
+
+	for i := 0; i < 10; i++ {
+		t.Logf("advance to epoch %d", i+1)
+		err := cs.advance()
+		require.NoError(t, err)
+		// wait for the coalescer to notify
+		time.Sleep(CoalesceMinDelay + 10*time.Millisecond)
+	}
+
+	t.Log("verifying index")
+	verifyIndex(t, cs, msgIndex)
+}
+
+func TestReorgMsgIndex(t *testing.T) {
+	// slightly more nuanced test that includes reorgs
+	// 1. Create an index with mock chain store
+	// 2. Advance/Reorg the chain for a few tipsets
+	// 3. Verify that the index contains all messages with the correct tipst/epoch
+	cs := newMockChainStore()
+	cs.genesis()
+
+	tmp := t.TempDir()
+	t.Cleanup(func() { _ = os.RemoveAll(tmp) })
+
+	msgIndex, err := NewMsgIndex(context.Background(), tmp, cs)
+	require.NoError(t, err)
+
+	defer msgIndex.Close() //nolint
+
+	for i := 0; i < 10; i++ {
+		t.Logf("advance to epoch %d", i+1)
+		err := cs.advance()
+		require.NoError(t, err)
+		// wait for the coalescer to notify
+		time.Sleep(CoalesceMinDelay + 10*time.Millisecond)
+	}
+
+	// a simple reorg
+	t.Log("doing reorg")
+	reorgme := cs.curTs
+	reorgmeParent, err := cs.GetTipSetFromKey(context.Background(), reorgme.Parents())
+	require.NoError(t, err)
+	cs.setHead(reorgmeParent)
+	reorgmeChild := cs.makeBlk()
+	err = cs.reorg([]*types.TipSet{reorgme}, []*types.TipSet{reorgmeChild})
+	require.NoError(t, err)
+	time.Sleep(CoalesceMinDelay + 10*time.Millisecond)
+
+	t.Log("verifying index")
+	verifyIndex(t, cs, msgIndex)
+
+	t.Log("verifying that reorged messages are not present")
+	verifyMissing(t, cs, msgIndex, reorgme)
+}
+
+func TestReconcileMsgIndex(t *testing.T) {
+	// test that exercises the reconciliation code paths
+	// 1. Create  and populate a basic msgindex, similar to TestBasicMsgIndex.
+	// 2. Close  it
+	// 3. Reorg the mock chain store
+	// 4. Reopen the index to trigger reconciliation
+	// 5. Enxure that only the stable messages remain.
+	cs := newMockChainStore()
+	cs.genesis()
+
+	tmp := t.TempDir()
+	t.Cleanup(func() { _ = os.RemoveAll(tmp) })
+
+	msgIndex, err := NewMsgIndex(context.Background(), tmp, cs)
+	require.NoError(t, err)
+
+	for i := 0; i < 10; i++ {
+		t.Logf("advance to epoch %d", i+1)
+		err := cs.advance()
+		require.NoError(t, err)
+		// wait for the coalescer to notify
+		time.Sleep(CoalesceMinDelay + 10*time.Millisecond)
+	}
+
+	// Close it and reorg
+	err = msgIndex.Close()
+	require.NoError(t, err)
+	cs.notify = nil
+
+	// a simple reorg
+	t.Log("doing reorg")
+	reorgme := cs.curTs
+	reorgmeParent, err := cs.GetTipSetFromKey(context.Background(), reorgme.Parents())
+	require.NoError(t, err)
+	cs.setHead(reorgmeParent)
+	reorgmeChild := cs.makeBlk()
+	err = cs.reorg([]*types.TipSet{reorgme}, []*types.TipSet{reorgmeChild})
+	require.NoError(t, err)
+
+	// reopen to reconcile
+	msgIndex, err = NewMsgIndex(context.Background(), tmp, cs)
+	require.NoError(t, err)
+
+	defer msgIndex.Close() //nolint
+
+	t.Log("verifying index")
+	// need to step one up because the last tipset is not known by the index
+	cs.setHead(reorgmeParent)
+	verifyIndex(t, cs, msgIndex)
+
+	t.Log("verifying that reorged and unknown messages are not present")
+	verifyMissing(t, cs, msgIndex, reorgme, reorgmeChild)
+}
+
+func verifyIndex(t *testing.T, cs *mockChainStore, msgIndex MsgIndex) {
+	for ts := cs.curTs; ts.Height() > 0; {
+		t.Logf("verify at height %d", ts.Height())
+		blks := ts.Blocks()
+		if len(blks) == 0 {
+			break
+		}
+
+		tsCid, err := ts.Key().Cid()
+		require.NoError(t, err)
+
+		msgs, err := cs.MessagesForTipset(context.Background(), ts)
+		require.NoError(t, err)
+		for _, m := range msgs {
+			minfo, _, err := msgIndex.GetMsgInfo(context.Background(), m.Cid())
+			require.NoError(t, err)
+			require.Equal(t, tsCid, minfo.TipSet)
+			require.Equal(t, ts.Height(), minfo.Epoch)
+		}
+
+		parents := ts.Parents()
+		ts, err = cs.GetTipSetFromKey(context.Background(), parents)
+		require.NoError(t, err)
+	}
+}
+
+func verifyMissing(t *testing.T, cs *mockChainStore, msgIndex MsgIndex, missing ...*types.TipSet) {
+	for _, ts := range missing {
+		msgs, err := cs.MessagesForTipset(context.Background(), ts)
+		require.NoError(t, err)
+		for _, m := range msgs {
+			_, _, err := msgIndex.GetMsgInfo(context.Background(), m.Cid())
+			require.Equal(t, ErrNotFound, err)
+		}
+	}
+}
+
+type mockChainStore struct {
+	notify store.ReorgNotifee
+
+	curTs   *types.TipSet
+	tipsets map[types.TipSetKey]*types.TipSet
+	msgs    map[types.TipSetKey][]types.ChainMsg
+
+	nonce uint64
+}
+
+var _ ChainStore = (*mockChainStore)(nil)
+
+var systemAddr address.Address
+var rng *rand.Rand
+
+func init() {
+	systemAddr, _ = address.NewIDAddress(0)
+	rng = rand.New(rand.NewSource(314159))
+
+	// adjust those to make tests snappy
+	CoalesceMinDelay = 100 * time.Millisecond
+	CoalesceMaxDelay = time.Second
+	CoalesceMergeInterval = 100 * time.Millisecond
+}
+
+func newMockChainStore() *mockChainStore {
+	return &mockChainStore{
+		tipsets: make(map[types.TipSetKey]*types.TipSet),
+		msgs:    make(map[types.TipSetKey][]types.ChainMsg),
+	}
+}
+
+func (cs *mockChainStore) genesis() {
+	genBlock := mock.MkBlock(nil, 0, 0)
+	genTs := mock.TipSet(genBlock)
+	cs.msgs[genTs.Key()] = nil
+	cs.setHead(genTs)
+}
+
+func (cs *mockChainStore) setHead(ts *types.TipSet) {
+	cs.curTs = ts
+	cs.tipsets[ts.Key()] = ts
+}
+
+func (cs *mockChainStore) advance() error {
+	ts := cs.makeBlk()
+	return cs.reorg(nil, []*types.TipSet{ts})
+}
+
+func (cs *mockChainStore) reorg(rev, app []*types.TipSet) error {
+	for _, ts := range rev {
+		parents := ts.Parents()
+		cs.curTs = cs.tipsets[parents]
+	}
+
+	for _, ts := range app {
+		cs.tipsets[ts.Key()] = ts
+		cs.curTs = ts
+	}
+
+	if cs.notify != nil {
+		return cs.notify(rev, app)
+	}
+
+	return nil
+}
+
+func (cs *mockChainStore) makeBlk() *types.TipSet {
+	height := cs.curTs.Height() + 1
+
+	blk := mock.MkBlock(cs.curTs, uint64(height), uint64(height))
+	blk.Messages = cs.makeGarbageCid()
+
+	ts := mock.TipSet(blk)
+	msg1 := cs.makeMsg()
+	msg2 := cs.makeMsg()
+	cs.msgs[ts.Key()] = []types.ChainMsg{msg1, msg2}
+
+	return ts
+}
+
+func (cs *mockChainStore) makeMsg() *types.Message {
+	nonce := cs.nonce
+	cs.nonce++
+	return &types.Message{To: systemAddr, From: systemAddr, Nonce: nonce}
+}
+
+func (cs *mockChainStore) makeGarbageCid() cid.Cid {
+	garbage := blocks.NewBlock([]byte{byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256))})
+	return garbage.Cid()
+}
+
+func (cs *mockChainStore) SubscribeHeadChanges(f store.ReorgNotifee) {
+	cs.notify = f
+}
+
+func (cs *mockChainStore) MessagesForTipset(ctx context.Context, ts *types.TipSet) ([]types.ChainMsg, error) {
+	msgs, ok := cs.msgs[ts.Key()]
+	if !ok {
+		return nil, errors.New("unknown tipset")
+	}
+
+	return msgs, nil
+}
+
+func (cs *mockChainStore) GetHeaviestTipSet() *types.TipSet {
+	return cs.curTs
+}
+
+func (cs *mockChainStore) GetTipSetFromKey(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
+	ts, ok := cs.tipsets[tsk]
+	if !ok {
+		return nil, errors.New("unknown tipset")
+	}
+	return ts, nil
+}

chain/messagepool/check.go

@@ -32,14 +32,14 @@ func (mp *MessagePool) CheckMessages(ctx context.Context, protos []*api.MessageP
 // CheckPendingMessages performs a set of logical sets for all messages pending from a given actor
 func (mp *MessagePool) CheckPendingMessages(ctx context.Context, from address.Address) ([][]api.MessageCheckStatus, error) {
 	var msgs []*types.Message
-	mp.lk.Lock()
+	mp.lk.RLock()
 	mset, ok := mp.pending[from]
 	if ok {
 		for _, sm := range mset.msgs {
 			msgs = append(msgs, &sm.Message)
 		}
 	}
-	mp.lk.Unlock()
+	mp.lk.RUnlock()
 
 	if len(msgs) == 0 {
 		return nil, nil
@@ -58,7 +58,7 @@ func (mp *MessagePool) CheckReplaceMessages(ctx context.Context, replace []*type
 	msgMap := make(map[address.Address]map[uint64]*types.Message)
 	count := 0
 
-	mp.lk.Lock()
+	mp.lk.RLock()
 	for _, m := range replace {
 		mmap, ok := msgMap[m.From]
 		if !ok {
@@ -76,7 +76,7 @@ func (mp *MessagePool) CheckReplaceMessages(ctx context.Context, replace []*type
 		}
 		mmap[m.Nonce] = m
 	}
-	mp.lk.Unlock()
+	mp.lk.RUnlock()
 
 	msgs := make([]*types.Message, 0, count)
 	start := 0
@@ -103,9 +103,9 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 	if mp.api.IsLite() {
 		return nil, nil
 	}
-	mp.curTsLk.Lock()
+	mp.curTsLk.RLock()
 	curTs := mp.curTs
-	mp.curTsLk.Unlock()
+	mp.curTsLk.RUnlock()
 
 	epoch := curTs.Height() + 1
 
@@ -143,7 +143,7 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 
 		st, ok := state[m.From]
 		if !ok {
-			mp.lk.Lock()
+			mp.lk.RLock()
 			mset, ok := mp.pending[m.From]
 			if ok && !interned {
 				st = &actorState{nextNonce: mset.nextNonce, requiredFunds: mset.requiredFunds}
@@ -151,14 +151,14 @@ func (mp *MessagePool) checkMessages(ctx context.Context, msgs []*types.Message,
 					st.requiredFunds = new(stdbig.Int).Add(st.requiredFunds, m.Message.Value.Int)
 				}
 				state[m.From] = st
-				mp.lk.Unlock()
+				mp.lk.RUnlock()
 
 				check.OK = true
 				check.Hint = map[string]interface{}{
 					"nonce": st.nextNonce,
 				}
 			} else {
-				mp.lk.Unlock()
+				mp.lk.RUnlock()
 
 				stateNonce, err := mp.getStateNonce(ctx, m.From, curTs)
 				if err != nil {

chain/messagepool/messagepool.go

@@ -118,7 +118,7 @@ func init() {
 }
 
 type MessagePool struct {
-	lk sync.Mutex
+	lk sync.RWMutex
 
 	ds dtypes.MetadataDS
 
@@ -137,9 +137,9 @@ type MessagePool struct {
 	// do NOT access this map directly, use getPendingMset, setPendingMset, deletePendingMset, forEachPending, and clearPending respectively
 	pending map[address.Address]*msgSet
 
-	keyCache map[address.Address]address.Address
+	keyCache *lru.Cache[address.Address, address.Address]
 
-	curTsLk sync.Mutex // DO NOT LOCK INSIDE lk
+	curTsLk sync.RWMutex // DO NOT LOCK INSIDE lk
 	curTs   *types.TipSet
 
 	cfgLk sync.RWMutex
@@ -372,6 +372,7 @@ func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.Upgra
 	cache, _ := lru.New2Q[cid.Cid, crypto.Signature](build.BlsSignatureCacheSize)
 	verifcache, _ := lru.New2Q[string, struct{}](build.VerifSigCacheSize)
 	noncecache, _ := lru.New[nonceCacheKey, uint64](256)
+	keycache, _ := lru.New[address.Address, address.Address](1_000_000)
 
 	cfg, err := loadConfig(ctx, ds)
 	if err != nil {
@@ -390,7 +391,7 @@ func New(ctx context.Context, api Provider, ds dtypes.MetadataDS, us stmgr.Upgra
 		repubTrigger:   make(chan struct{}, 1),
 		localAddrs:     make(map[address.Address]struct{}),
 		pending:        make(map[address.Address]*msgSet),
-		keyCache:       make(map[address.Address]address.Address),
+		keyCache:       keycache,
 		minGasPrice:    types.NewInt(0),
 		getNtwkVersion: us.GetNtwkVersion,
 		pruneTrigger:   make(chan struct{}, 1),
@@ -474,8 +475,8 @@ func (mp *MessagePool) TryForEachPendingMessage(f func(cid.Cid) error) error {
 
 func (mp *MessagePool) resolveToKey(ctx context.Context, addr address.Address) (address.Address, error) {
 	// check the cache
-	a, f := mp.keyCache[addr]
-	if f {
+	a, ok := mp.keyCache.Get(addr)
+	if ok {
 		return a, nil
 	}
 
@@ -486,8 +487,8 @@ func (mp *MessagePool) resolveToKey(ctx context.Context, addr address.Address) (
 	}
 
 	// place both entries in the cache (may both be key addresses, which is fine)
-	mp.keyCache[addr] = ka
-	mp.keyCache[ka] = ka
+	mp.keyCache.Add(addr, ka)
+	mp.keyCache.Add(ka, ka)
 
 	return ka, nil
 }
@@ -763,7 +764,28 @@ func (mp *MessagePool) Add(ctx context.Context, m *types.SignedMessage) error {
 		<-mp.addSema
 	}()
 
+	mp.curTsLk.RLock()
+	tmpCurTs := mp.curTs
+	mp.curTsLk.RUnlock()
+
+	//ensures computations are cached without holding lock
+	_, _ = mp.api.GetActorAfter(m.Message.From, tmpCurTs)
+	_, _ = mp.getStateNonce(ctx, m.Message.From, tmpCurTs)
+
 	mp.curTsLk.Lock()
+	if tmpCurTs == mp.curTs {
+		//with the lock enabled, mp.curTs is the same Ts as we just had, so we know that our computations are cached
+	} else {
+		//curTs has been updated so we want to cache the new one:
+		tmpCurTs = mp.curTs
+		//we want to release the lock, cache the computations then grab it again
+		mp.curTsLk.Unlock()
+		_, _ = mp.api.GetActorAfter(m.Message.From, tmpCurTs)
+		_, _ = mp.getStateNonce(ctx, m.Message.From, tmpCurTs)
+		mp.curTsLk.Lock()
+		//now that we have the lock, we continue, we could do this as a loop forever, but that's bad to loop forever, and this was added as an optimization and it seems once is enough because the computation < block time
+	}
+
 	defer mp.curTsLk.Unlock()
 
 	_, err = mp.addTs(ctx, m, mp.curTs, false, false)
@@ -852,9 +874,6 @@ func (mp *MessagePool) addTs(ctx context.Context, m *types.SignedMessage, curTs
 		return false, xerrors.Errorf("minimum expected nonce is %d: %w", snonce, ErrNonceTooLow)
 	}
 
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
-
 	senderAct, err := mp.api.GetActorAfter(m.Message.From, curTs)
 	if err != nil {
 		return false, xerrors.Errorf("failed to get sender actor: %w", err)
@@ -869,6 +888,9 @@ func (mp *MessagePool) addTs(ctx context.Context, m *types.SignedMessage, curTs
 		return false, xerrors.Errorf("sender actor %s is not a valid top-level sender", m.Message.From)
 	}
 
+	mp.lk.Lock()
+	defer mp.lk.Unlock()
+
 	publish, err := mp.verifyMsgBeforeAdd(ctx, m, curTs, local)
 	if err != nil {
 		return false, xerrors.Errorf("verify msg failed: %w", err)
@@ -1001,19 +1023,19 @@ func (mp *MessagePool) addLocked(ctx context.Context, m *types.SignedMessage, st
 }
 
 func (mp *MessagePool) GetNonce(ctx context.Context, addr address.Address, _ types.TipSetKey) (uint64, error) {
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
+	mp.curTsLk.RLock()
+	defer mp.curTsLk.RUnlock()
 
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
+	mp.lk.RLock()
+	defer mp.lk.RUnlock()
 
 	return mp.getNonceLocked(ctx, addr, mp.curTs)
 }
 
 // GetActor should not be used. It is only here to satisfy interface mess caused by lite node handling
 func (mp *MessagePool) GetActor(_ context.Context, addr address.Address, _ types.TipSetKey) (*types.Actor, error) {
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
+	mp.curTsLk.RLock()
+	defer mp.curTsLk.RUnlock()
 	return mp.api.GetActorAfter(addr, mp.curTs)
 }
 
@@ -1164,11 +1186,11 @@ func (mp *MessagePool) remove(ctx context.Context, from address.Address, nonce u
 }
 
 func (mp *MessagePool) Pending(ctx context.Context) ([]*types.SignedMessage, *types.TipSet) {
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
+	mp.curTsLk.RLock()
+	defer mp.curTsLk.RUnlock()
 
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
+	mp.lk.RLock()
+	defer mp.lk.RUnlock()
 
 	return mp.allPending(ctx)
 }
@@ -1184,11 +1206,11 @@ func (mp *MessagePool) allPending(ctx context.Context) ([]*types.SignedMessage,
 }
 
 func (mp *MessagePool) PendingFor(ctx context.Context, a address.Address) ([]*types.SignedMessage, *types.TipSet) {
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
+	mp.curTsLk.RLock()
+	defer mp.curTsLk.RUnlock()
 
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
+	mp.lk.RLock()
+	defer mp.lk.RUnlock()
 	return mp.pendingFor(ctx, a), mp.curTs
 }
 
@@ -1237,9 +1259,9 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 
 	maybeRepub := func(cid cid.Cid) {
 		if !repubTrigger {
-			mp.lk.Lock()
+			mp.lk.RLock()
 			_, republished := mp.republished[cid]
-			mp.lk.Unlock()
+			mp.lk.RUnlock()
 			if republished {
 				repubTrigger = true
 			}
@@ -1310,9 +1332,9 @@ func (mp *MessagePool) HeadChange(ctx context.Context, revert []*types.TipSet, a
 	}
 
 	if len(revert) > 0 && futureDebug {
-		mp.lk.Lock()
+		mp.lk.RLock()
 		msgs, ts := mp.allPending(ctx)
-		mp.lk.Unlock()
+		mp.lk.RUnlock()
 
 		buckets := map[address.Address]*statBucket{}
 

chain/messagepool/repub.go

@@ -20,17 +20,18 @@ const repubMsgLimit = 30
 var RepublishBatchDelay = 100 * time.Millisecond
 
 func (mp *MessagePool) republishPendingMessages(ctx context.Context) error {
-	mp.curTsLk.Lock()
+	mp.curTsLk.RLock()
 	ts := mp.curTs
 
 	baseFee, err := mp.api.ChainComputeBaseFee(context.TODO(), ts)
+	mp.curTsLk.RUnlock()
 	if err != nil {
-		mp.curTsLk.Unlock()
 		return xerrors.Errorf("computing basefee: %w", err)
 	}
 	baseFeeLowerBound := getBaseFeeLowerBound(baseFee, baseFeeLowerBoundFactor)
 
 	pending := make(map[address.Address]map[uint64]*types.SignedMessage)
+	mp.curTsLk.Lock()
 	mp.lk.Lock()
 	mp.republished = nil // clear this to avoid races triggering an early republish
 	mp.forEachLocal(ctx, func(ctx context.Context, actor address.Address) {

chain/messagepool/selection.go

@@ -43,6 +43,7 @@ func (mp *MessagePool) SelectMessages(ctx context.Context, ts *types.TipSet, tq
 	mp.curTsLk.Lock()
 	defer mp.curTsLk.Unlock()
 
+	//TODO confirm if we can switch to RLock here for performance
 	mp.lk.Lock()
 	defer mp.lk.Unlock()
 

chain/stmgr/call.go

@@ -52,8 +52,8 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 }
 
 // CallWithGas calculates the state for a given tipset, and then applies the given message on top of that state.
-func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
-	return sm.callInternal(ctx, msg, priorMsgs, ts, cid.Undef, sm.GetNetworkVersion, true, true)
+func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet, applyTsMessages bool) (*api.InvocResult, error) {
+	return sm.callInternal(ctx, msg, priorMsgs, ts, cid.Undef, sm.GetNetworkVersion, true, applyTsMessages)
 }
 
 // CallAtStateAndVersion allows you to specify a message to execute on the given stateCid and network version.
@@ -117,12 +117,22 @@ func (sm *StateManager) callInternal(ctx context.Context, msg *types.Message, pr
 	if stateCid == cid.Undef {
 		stateCid = ts.ParentState()
 	}
+	tsMsgs, err := sm.cs.MessagesForTipset(ctx, ts)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to lookup messages for parent tipset: %w", err)
+	}
+
 	if applyTsMessages {
-		tsMsgs, err := sm.cs.MessagesForTipset(ctx, ts)
-		if err != nil {
-			return nil, xerrors.Errorf("failed to lookup messages for parent tipset: %w", err)
-		}
 		priorMsgs = append(tsMsgs, priorMsgs...)
+	} else {
+		var filteredTsMsgs []types.ChainMsg
+		for _, tsMsg := range tsMsgs {
+			//TODO we should technically be normalizing the filecoin address of from when we compare here
+			if tsMsg.VMMessage().From == msg.VMMessage().From {
+				filteredTsMsgs = append(filteredTsMsgs, tsMsg)
+			}
+		}
+		priorMsgs = append(filteredTsMsgs, priorMsgs...)
 	}
 
 	// Technically, the tipset we're passing in here should be ts+1, but that may not exist.

chain/stmgr/execute.go

@@ -128,10 +128,43 @@ func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types
 }
 
 func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (cid.Cid, []*api.InvocResult, error) {
+	tsKey := ts.Key()
+
+	// check if we have the trace for this tipset in the cache
+	sm.execTraceCacheLock.Lock()
+	if entry, ok := sm.execTraceCache.Get(tsKey); ok {
+		// we have to make a deep copy since caller can modify the invocTrace
+		// and we don't want that to change what we store in cache
+		invocTraceCopy := makeDeepCopy(entry.invocTrace)
+		sm.execTraceCacheLock.Unlock()
+		return entry.postStateRoot, invocTraceCopy, nil
+	}
+	sm.execTraceCacheLock.Unlock()
+
 	var invocTrace []*api.InvocResult
 	st, err := sm.ExecutionTraceWithMonitor(ctx, ts, &InvocationTracer{trace: &invocTrace})
 	if err != nil {
 		return cid.Undef, nil, err
 	}
+
+	invocTraceCopy := makeDeepCopy(invocTrace)
+
+	sm.execTraceCacheLock.Lock()
+	sm.execTraceCache.Add(tsKey, tipSetCacheEntry{st, invocTraceCopy})
+	sm.execTraceCacheLock.Unlock()
+
 	return st, invocTrace, nil
 }
+
+func makeDeepCopy(invocTrace []*api.InvocResult) []*api.InvocResult {
+	c := make([]*api.InvocResult, len(invocTrace))
+	for i, ir := range invocTrace {
+		if ir == nil {
+			continue
+		}
+		tmp := *ir
+		c[i] = &tmp
+	}
+
+	return c
+}

chain/stmgr/forks_test.go

@@ -36,6 +36,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/gen"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	. "github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -168,7 +169,7 @@ func TestForkHeightTriggers(t *testing.T) {
 				}
 
 				return st.Flush(ctx)
-			}}}, cg.BeaconSchedule(), datastore.NewMapDatastore())
+			}}}, cg.BeaconSchedule(), datastore.NewMapDatastore(), index.DummyMsgIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -286,7 +287,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 				root cid.Cid, height abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
 				migrationCount++
 				return root, nil
-			}}}, cg.BeaconSchedule(), datastore.NewMapDatastore())
+			}}}, cg.BeaconSchedule(), datastore.NewMapDatastore(), index.DummyMsgIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -340,7 +341,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 		currentHeight := ts.TipSet.TipSet().Height()
 
 		// CallWithGas calls on top of the given tipset.
-		ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet())
+		ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet(), true)
 		if parentHeight <= testForkHeight && currentHeight >= testForkHeight {
 			// If I had a fork, or I _will_ have a fork, it should fail.
 			require.Equal(t, ErrExpensiveFork, err)
@@ -361,7 +362,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 		// Calls without a tipset should walk back to the last non-fork tipset.
 		// We _verify_ that the migration wasn't run multiple times at the end of the
 		// test.
-		ret, err = sm.CallWithGas(ctx, m, nil, nil)
+		ret, err = sm.CallWithGas(ctx, m, nil, nil, true)
 		require.NoError(t, err)
 		require.True(t, ret.MsgRct.ExitCode.IsSuccess())
 
@@ -504,7 +505,7 @@ func TestForkPreMigration(t *testing.T) {
 					return nil
 				},
 			}}},
-		}, cg.BeaconSchedule(), datastore.NewMapDatastore())
+		}, cg.BeaconSchedule(), datastore.NewMapDatastore(), index.DummyMsgIndex)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -579,6 +580,7 @@ func TestDisablePreMigration(t *testing.T) {
 		},
 		cg.BeaconSchedule(),
 		datastore.NewMapDatastore(),
+		index.DummyMsgIndex,
 	)
 	require.NoError(t, err)
 	require.NoError(t, sm.Start(context.Background()))
@@ -633,6 +635,7 @@ func TestMigrtionCache(t *testing.T) {
 		},
 		cg.BeaconSchedule(),
 		metadataDs,
+		index.DummyMsgIndex,
 	)
 	require.NoError(t, err)
 	require.NoError(t, sm.Start(context.Background()))
@@ -685,6 +688,7 @@ func TestMigrtionCache(t *testing.T) {
 			},
 			cg.BeaconSchedule(),
 			metadataDs,
+			index.DummyMsgIndex,
 		)
 		require.NoError(t, err)
 		sm.SetVMConstructor(func(ctx context.Context, vmopt *vm.VMOpts) (vm.Interface, error) {

chain/stmgr/searchwait.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -18,6 +19,7 @@ import (
 // happened, with an optional limit to how many epochs it will search. It guarantees that the message has been on
 // chain for at least confidence epochs without being reverted before returning.
 func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confidence uint64, lookbackLimit abi.ChainEpoch, allowReplaced bool) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) {
+	// TODO use the index to speed this up.
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -55,10 +57,15 @@ func (sm *StateManager) WaitForMessage(ctx context.Context, mcid cid.Cid, confid
 	var backFm cid.Cid
 	backSearchWait := make(chan struct{})
 	go func() {
-		fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head[0].Val, msg, lookbackLimit, allowReplaced)
-		if err != nil {
-			log.Warnf("failed to look back through chain for message: %v", err)
-			return
+		fts, r, foundMsg, err := sm.searchForIndexedMsg(ctx, mcid, msg)
+
+		found := (err == nil && r != nil && foundMsg.Defined())
+		if !found {
+			fts, r, foundMsg, err = sm.searchBackForMsg(ctx, head[0].Val, msg, lookbackLimit, allowReplaced)
+			if err != nil {
+				log.Warnf("failed to look back through chain for message: %v", err)
+				return
+			}
 		}
 
 		backTs = fts
@@ -145,7 +152,30 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, head *types.TipSet
 		return head, r, foundMsg, nil
 	}
 
-	fts, r, foundMsg, err := sm.searchBackForMsg(ctx, head, msg, lookbackLimit, allowReplaced)
+	fts, r, foundMsg, err := sm.searchForIndexedMsg(ctx, mcid, msg)
+
+	switch {
+	case err == nil:
+		if r != nil && foundMsg.Defined() {
+			return fts, r, foundMsg, nil
+		}
+
+		// debug log this, it's noteworthy
+		if r == nil {
+			log.Debugf("missing receipt for message in index for %s", mcid)
+		}
+		if !foundMsg.Defined() {
+			log.Debugf("message %s not found", mcid)
+		}
+
+	case errors.Is(err, index.ErrNotFound):
+		// ok for the index to have incomplete data
+
+	default:
+		log.Warnf("error searching message index: %s", err)
+	}
+
+	fts, r, foundMsg, err = sm.searchBackForMsg(ctx, head, msg, lookbackLimit, allowReplaced)
 
 	if err != nil {
 		log.Warnf("failed to look back through chain for message %s", mcid)
@@ -159,6 +189,52 @@ func (sm *StateManager) SearchForMessage(ctx context.Context, head *types.TipSet
 	return fts, r, foundMsg, nil
 }
 
+func (sm *StateManager) searchForIndexedMsg(ctx context.Context, mcid cid.Cid, m types.ChainMsg) (*types.TipSet, *types.MessageReceipt, cid.Cid, error) {
+	minfo, xtsCid, err := sm.msgIndex.GetMsgInfo(ctx, mcid)
+	if err != nil {
+		return nil, nil, cid.Undef, xerrors.Errorf("error looking up message in index: %w", err)
+	}
+
+	// check the height against the current tipset; minimum execution confidence requires that the
+	// inclusion tipset height is lower than the current head + 1
+	curTs := sm.cs.GetHeaviestTipSet()
+	if curTs.Height() <= minfo.Epoch+1 {
+		return nil, nil, cid.Undef, xerrors.Errorf("indexed message does not appear before the current tipset; index epoch: %d, current epoch: %d", minfo.Epoch, curTs.Height())
+	}
+
+	// now get the execution tipset
+	var xts *types.TipSet = nil
+	if xtsCid != cid.Undef {
+		// lookup by cid which is faster
+		xts, err = sm.cs.GetTipSetByCid(ctx, xtsCid)
+		if err != nil {
+			return nil, nil, cid.Undef, xerrors.Errorf("error calling GetTipSetByCid: %w", err)
+		}
+	} else {
+		xts, err = sm.cs.GetTipsetByHeight(ctx, minfo.Epoch+1, curTs, false)
+		if err != nil {
+			return nil, nil, cid.Undef, xerrors.Errorf("error calling GetTipsetByHeight: %w", err)
+		}
+	}
+
+	// check that the parent of the execution index is indeed the inclusion tipset
+	parent := xts.Parents()
+	parentCid, err := parent.Cid()
+	if err != nil {
+		return nil, nil, cid.Undef, xerrors.Errorf("error computing tipset cid: %w", err)
+	}
+
+	if !parentCid.Equals(minfo.TipSet) {
+		return nil, nil, cid.Undef, xerrors.Errorf("inclusion tipset mismatch: have %s, expected %s", parentCid, minfo.TipSet)
+	}
+
+	r, foundMsg, err := sm.tipsetExecutedMessage(ctx, xts, mcid, m.VMMessage(), false)
+	if err != nil {
+		return nil, nil, cid.Undef, xerrors.Errorf("error in tipstExecutedMessage: %w", err)
+	}
+	return xts, r, foundMsg, nil
+}
+
 // searchBackForMsg searches up to limit tipsets backwards from the given
 // tipset for a message receipt.
 // If limit is

chain/stmgr/stmgr.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sync"
 
+	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/ipfs/go-cid"
 	dstore "github.com/ipfs/go-datastore"
 	cbor "github.com/ipfs/go-ipld-cbor"
@@ -25,6 +26,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/beacon"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/rand"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -38,6 +40,8 @@ import (
 const LookbackNoLimit = api.LookbackNoLimit
 const ReceiptAmtBitwidth = 3
 
+const execTraceCacheSize = 16
+
 var log = logging.Logger("statemgr")
 
 type StateManagerAPI interface {
@@ -130,12 +134,20 @@ type StateManager struct {
 	postIgnitionVesting []msig0.State
 	postCalicoVesting   []msig0.State
 
-	genesisPledge      abi.TokenAmount
-	genesisMarketFunds abi.TokenAmount
+	genesisPledge abi.TokenAmount
 
 	tsExec        Executor
 	tsExecMonitor ExecMonitor
 	beacon        beacon.Schedule
+
+	msgIndex index.MsgIndex
+
+	// We keep a small cache for calls to ExecutionTrace which helps improve
+	// performance for node operators like exchanges and block explorers
+	execTraceCache *lru.ARCCache[types.TipSetKey, tipSetCacheEntry]
+	// We need a lock while making the copy as to prevent other callers
+	// overwrite the cache while making the copy
+	execTraceCacheLock sync.Mutex
 }
 
 // Caches a single state tree
@@ -144,7 +156,12 @@ type treeCache struct {
 	tree *state.StateTree
 }
 
-func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule, metadataDs dstore.Batching) (*StateManager, error) {
+type tipSetCacheEntry struct {
+	postStateRoot cid.Cid
+	invocTrace    []*api.InvocResult
+}
+
+func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, beacon beacon.Schedule, metadataDs dstore.Batching, msgIndex index.MsgIndex) (*StateManager, error) {
 	// If we have upgrades, make sure they're in-order and make sense.
 	if err := us.Validate(); err != nil {
 		return nil, err
@@ -183,6 +200,11 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
 		}
 	}
 
+	execTraceCache, err := lru.NewARC[types.TipSetKey, tipSetCacheEntry](execTraceCacheSize)
+	if err != nil {
+		return nil, err
+	}
+
 	return &StateManager{
 		networkVersions:   networkVersions,
 		latestVersion:     lastVersion,
@@ -198,12 +220,14 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder,
 			root: cid.Undef,
 			tree: nil,
 		},
-		compWait: make(map[string]chan struct{}),
+		compWait:       make(map[string]chan struct{}),
+		msgIndex:       msgIndex,
+		execTraceCache: execTraceCache,
 	}, nil
 }
 
-func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor, metadataDs dstore.Batching) (*StateManager, error) {
-	sm, err := NewStateManager(cs, exec, sys, us, b, metadataDs)
+func NewStateManagerWithUpgradeScheduleAndMonitor(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, us UpgradeSchedule, b beacon.Schedule, em ExecMonitor, metadataDs dstore.Batching, msgIndex index.MsgIndex) (*StateManager, error) {
+	sm, err := NewStateManager(cs, exec, sys, us, b, metadataDs, msgIndex)
 	if err != nil {
 		return nil, err
 	}

chain/stmgr/supply.go

@@ -51,17 +51,13 @@ func (sm *StateManager) setupGenesisVestingSchedule(ctx context.Context) error {
 		return xerrors.Errorf("loading state tree: %w", err)
 	}
 
-	gmf, err := getFilMarketLocked(ctx, sTree)
-	if err != nil {
-		return xerrors.Errorf("setting up genesis market funds: %w", err)
-	}
-
 	gp, err := getFilPowerLocked(ctx, sTree)
 	if err != nil {
 		return xerrors.Errorf("setting up genesis pledge: %w", err)
 	}
 
-	sm.genesisMarketFunds = gmf
+	sm.genesisMsigLk.Lock()
+	defer sm.genesisMsigLk.Unlock()
 	sm.genesisPledge = gp
 
 	totalsByEpoch := make(map[abi.ChainEpoch]abi.TokenAmount)
@@ -128,6 +124,8 @@ func (sm *StateManager) setupPostIgnitionVesting(ctx context.Context) error {
 	totalsByEpoch[sixYears] = big.NewInt(100_000_000)
 	totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000))
 
+	sm.genesisMsigLk.Lock()
+	defer sm.genesisMsigLk.Unlock()
 	sm.postIgnitionVesting = make([]msig0.State, 0, len(totalsByEpoch))
 	for k, v := range totalsByEpoch {
 		ns := msig0.State{
@@ -178,6 +176,9 @@ func (sm *StateManager) setupPostCalicoVesting(ctx context.Context) error {
 	totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(300_000_000))
 	totalsByEpoch[sixYears] = big.Add(totalsByEpoch[sixYears], big.NewInt(9_805_053))
 
+	sm.genesisMsigLk.Lock()
+	defer sm.genesisMsigLk.Unlock()
+
 	sm.postCalicoVesting = make([]msig0.State, 0, len(totalsByEpoch))
 	for k, v := range totalsByEpoch {
 		ns := msig0.State{
@@ -198,21 +199,20 @@ func (sm *StateManager) setupPostCalicoVesting(ctx context.Context) error {
 func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch) (abi.TokenAmount, error) {
 	vf := big.Zero()
 
-	sm.genesisMsigLk.Lock()
-	defer sm.genesisMsigLk.Unlock()
-
 	// TODO: combine all this?
-	if sm.preIgnitionVesting == nil || sm.genesisPledge.IsZero() || sm.genesisMarketFunds.IsZero() {
+	if sm.preIgnitionVesting == nil || sm.genesisPledge.IsZero() {
 		err := sm.setupGenesisVestingSchedule(ctx)
 		if err != nil {
 			return vf, xerrors.Errorf("failed to setup pre-ignition vesting schedule: %w", err)
 		}
+
 	}
 	if sm.postIgnitionVesting == nil {
 		err := sm.setupPostIgnitionVesting(ctx)
 		if err != nil {
 			return vf, xerrors.Errorf("failed to setup post-ignition vesting schedule: %w", err)
 		}
+
 	}
 	if sm.postCalicoVesting == nil {
 		err := sm.setupPostCalicoVesting(ctx)
@@ -246,8 +246,6 @@ func (sm *StateManager) GetFilVested(ctx context.Context, height abi.ChainEpoch)
 	if height <= build.UpgradeAssemblyHeight {
 		// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
 		vf = big.Add(vf, sm.genesisPledge)
-		// continue to use preIgnitionGenInfos, nothing changed at the Ignition epoch
-		vf = big.Add(vf, sm.genesisMarketFunds)
 	}
 
 	return vf, nil

chain/store/index.go

@@ -6,10 +6,11 @@ import (
 	"strconv"
 	"sync"
 
+	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
-
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -30,16 +31,20 @@ type ChainIndex struct {
 	indexCacheLk sync.Mutex
 	indexCache   map[types.TipSetKey]*lbEntry
 
-	loadTipSet loadTipSetFunc
+	loadTipSet   loadTipSetFunc
+	lookupTipSet lookupTipSetCIDFunc
 
 	skipLength abi.ChainEpoch
 }
 type loadTipSetFunc func(context.Context, types.TipSetKey) (*types.TipSet, error)
 
-func NewChainIndex(lts loadTipSetFunc) *ChainIndex {
+type lookupTipSetCIDFunc func(ctx context.Context, epoch abi.ChainEpoch) (*cid.Cid, error)
+
+func NewChainIndex(lts loadTipSetFunc, luts lookupTipSetCIDFunc) *ChainIndex {
 	return &ChainIndex{
 		indexCache: make(map[types.TipSetKey]*lbEntry, DefaultChainIndexCacheSize),
 		loadTipSet: lts,
+		lookupTipSet:  luts,
 		skipLength: 20,
 	}
 }
@@ -50,6 +55,25 @@ type lbEntry struct {
 }
 
 func (ci *ChainIndex) GetTipsetByHeight(ctx context.Context, from *types.TipSet, to abi.ChainEpoch) (*types.TipSet, error) {
+	if ci.lookupTipSet != nil {
+		tsc, err := ci.lookupTipSet(ctx, to)
+		switch {
+		case err == index.ErrNotFound:
+			// fall through
+		case err != nil:
+			return nil, xerrors.Errorf("failed to load tipset cid: %w", err)
+		default:
+			ts, err := ci.loadTipSet(ctx, types.NewTipSetKey(*tsc))
+			if err != nil {
+				return nil, xerrors.Errorf("failed to load tipset: %w", err)
+			}
+			// make sure that the tipset is correct!
+			if ts.Height() == to {
+				return ts, nil
+			}
+			// otherwise, fall through
+		}
+	}
 	if from.Height()-to <= ci.skipLength {
 		return ci.walkBack(ctx, from, to)
 	}

chain/store/messages.go

@@ -114,12 +114,35 @@ func (cs *ChainStore) BlockMsgsForTipset(ctx context.Context, ts *types.TipSet)
 		return nil, xerrors.Errorf("failed to load state tree at tipset %s: %w", ts, err)
 	}
 
+	useIds := false
 	selectMsg := func(m *types.Message) (bool, error) {
 		var sender address.Address
 		if ts.Height() >= build.UpgradeHyperdriveHeight {
-			sender, err = st.LookupID(m.From)
-			if err != nil {
-				return false, err
+			if useIds {
+				sender, err = st.LookupID(m.From)
+				if err != nil {
+					return false, xerrors.Errorf("failed to resolve sender: %w", err)
+				}
+			} else {
+				if m.From.Protocol() != address.ID {
+					// we haven't been told to use IDs, just use the robust addr
+					sender = m.From
+				} else {
+					// uh-oh, we actually have an ID-sender!
+					useIds = true
+					for robust, nonce := range applied {
+						resolved, err := st.LookupID(robust)
+						if err != nil {
+							return false, xerrors.Errorf("failed to resolve sender: %w", err)
+						}
+						applied[resolved] = nonce
+					}
+
+					sender, err = st.LookupID(m.From)
+					if err != nil {
+						return false, xerrors.Errorf("failed to resolve sender: %w", err)
+					}
+				}
 			}
 		} else {
 			sender = m.From

chain/store/snapshot.go

@@ -3,12 +3,14 @@ package store
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"sync"
 	"time"
 
 	"github.com/ipfs/go-cid"
+	format "github.com/ipfs/go-ipld-format"
 	blocks "github.com/ipfs/go-libipfs/blocks"
 	"github.com/ipld/go-car"
 	carutil "github.com/ipld/go-car/util"
@@ -167,8 +169,11 @@ func (t walkSchedTaskType) String() string {
 }
 
 type walkTask struct {
-	c        cid.Cid
-	taskType walkSchedTaskType
+	c                cid.Cid
+	taskType         walkSchedTaskType
+	topLevelTaskType walkSchedTaskType
+	blockCid         cid.Cid
+	epoch            abi.ChainEpoch
 }
 
 // an ever growing FIFO
@@ -317,8 +322,11 @@ func newWalkScheduler(ctx context.Context, store bstore.Blockstore, cfg walkSche
 			cancel() // kill workers
 			return nil, ctx.Err()
 		case s.workerTasks.in <- walkTask{
-			c:        b.Cid(),
-			taskType: blockTask,
+			c:                b.Cid(),
+			taskType:         blockTask,
+			topLevelTaskType: blockTask,
+			blockCid:         b.Cid(),
+			epoch:            cfg.head.Height(),
 		}:
 		}
 	}
@@ -416,8 +424,17 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 	}
 
 	blk, err := s.store.Get(s.ctx, t.c)
+	if errors.Is(err, format.ErrNotFound{}) && t.topLevelTaskType == receiptTask {
+		log.Debugw("ignoring not-found block in Receipts",
+			"block", t.blockCid,
+			"epoch", t.epoch,
+			"cid", t.c)
+		return nil
+	}
 	if err != nil {
-		return xerrors.Errorf("writing object to car, bs.Get: %w", err)
+		return xerrors.Errorf(
+			"blockstore.Get(%s). Task: %s. Block: %s (%s). Epoch: %d. Err: %w",
+			t.c, t.taskType, t.topLevelTaskType, t.blockCid, t.epoch, err)
 	}
 
 	s.results <- taskResult{
@@ -427,13 +444,8 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 
 	// extract relevant dags to walk from the block
 	if t.taskType == blockTask {
-		blk := t.c
-		data, err := s.store.Get(s.ctx, blk)
-		if err != nil {
-			return err
-		}
 		var b types.BlockHeader
-		if err := b.UnmarshalCBOR(bytes.NewBuffer(data.RawData())); err != nil {
+		if err := b.UnmarshalCBOR(bytes.NewBuffer(blk.RawData())); err != nil {
 			return xerrors.Errorf("unmarshalling block header (cid=%s): %w", blk, err)
 		}
 		if b.Height%1_000 == 0 {
@@ -443,13 +455,19 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 			log.Info("exporting genesis block")
 			for i := range b.Parents {
 				s.enqueueIfNew(walkTask{
-					c:        b.Parents[i],
-					taskType: dagTask,
+					c:                b.Parents[i],
+					taskType:         dagTask,
+					topLevelTaskType: blockTask,
+					blockCid:         b.Parents[i],
+					epoch:            0,
 				})
 			}
 			s.enqueueIfNew(walkTask{
-				c:        b.ParentStateRoot,
-				taskType: stateTask,
+				c:                b.ParentStateRoot,
+				taskType:         stateTask,
+				topLevelTaskType: stateTask,
+				blockCid:         t.c,
+				epoch:            0,
 			})
 
 			return s.sendFinish(workerN)
@@ -457,33 +475,45 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 		// enqueue block parents
 		for i := range b.Parents {
 			s.enqueueIfNew(walkTask{
-				c:        b.Parents[i],
-				taskType: blockTask,
+				c:                b.Parents[i],
+				taskType:         blockTask,
+				topLevelTaskType: blockTask,
+				blockCid:         b.Parents[i],
+				epoch:            b.Height,
 			})
 		}
 		if s.cfg.tail.Height() >= b.Height {
-			log.Debugw("tail reached: only blocks will be exported from now until genesis", "cid", blk.String())
+			log.Debugw("tail reached: only blocks will be exported from now until genesis", "cid", t.c.String())
 			return nil
 		}
 
 		if s.cfg.includeMessages {
 			// enqueue block messages
 			s.enqueueIfNew(walkTask{
-				c:        b.Messages,
-				taskType: messageTask,
+				c:                b.Messages,
+				taskType:         messageTask,
+				topLevelTaskType: messageTask,
+				blockCid:         t.c,
+				epoch:            b.Height,
 			})
 		}
 		if s.cfg.includeReceipts {
 			// enqueue block receipts
 			s.enqueueIfNew(walkTask{
-				c:        b.ParentMessageReceipts,
-				taskType: receiptTask,
+				c:                b.ParentMessageReceipts,
+				taskType:         receiptTask,
+				topLevelTaskType: receiptTask,
+				blockCid:         t.c,
+				epoch:            b.Height,
 			})
 		}
 		if s.cfg.includeState {
 			s.enqueueIfNew(walkTask{
-				c:        b.ParentStateRoot,
-				taskType: stateTask,
+				c:                b.ParentStateRoot,
+				taskType:         stateTask,
+				topLevelTaskType: stateTask,
+				blockCid:         t.c,
+				epoch:            b.Height,
 			})
 		}
 
@@ -497,8 +527,11 @@ func (s *walkScheduler) processTask(t walkTask, workerN int) error {
 		}
 
 		s.enqueueIfNew(walkTask{
-			c:        c,
-			taskType: dagTask,
+			c:                c,
+			taskType:         dagTask,
+			topLevelTaskType: t.topLevelTaskType,
+			blockCid:         t.blockCid,
+			epoch:            t.epoch,
 		})
 	})
 }

chain/store/store.go

@@ -161,7 +161,8 @@ func NewChainStore(chainBs bstore.Blockstore, stateBs bstore.Blockstore, ds dsto
 		evtTypeHeadChange: j.RegisterEventType("sync", "head_change"),
 	}
 
-	ci := NewChainIndex(cs.LoadTipSet)
+	// TODO: what's the best way to get the message (tipset) index loaded into here?
+	ci := NewChainIndex(cs.LoadTipSet, nil)
 
 	cs.cindex = ci
 

chain/store/store_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/gen"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -214,7 +215,7 @@ func TestChainExportImportFull(t *testing.T) {
 		t.Fatal("imported chain differed from exported chain")
 	}
 
-	sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule(), ds)
+	sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), nil, filcns.DefaultUpgradeSchedule(), cg.BeaconSchedule(), ds, index.DummyMsgIndex)
 	if err != nil {
 		t.Fatal(err)
 	}

chain/sub/bcast/consistent.go

@@ -0,0 +1,197 @@
+package bcast
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+var log = logging.Logger("sub-cb")
+
+const (
+	// GcSanityCheck determines the number of epochs in the past
+	// that will be garbage collected from the current epoch.
+	GcSanityCheck = 100
+	// GcLookback determines the number of epochs kept in the consistent
+	// broadcast cache.
+	GcLookback = 5
+	// GcDeepCheck determines the number of epochs in the past that we
+	// we try cleaning in the deep garbage collection round.
+	GcDeepCheck = 2880 // (24h*60m*60s)/30s per epoch
+	// GcDeepInterval determines after the number of epochs for which
+	// we are going to start a deeper garbage collection round.
+	GcDeepInterval = 1000
+)
+
+type blksInfo struct {
+	ctx    context.Context
+	cancel context.CancelFunc
+	blks   []cid.Cid
+}
+
+type bcastDict struct {
+	m map[string]*blksInfo
+}
+
+func (bd *bcastDict) load(key []byte) (*blksInfo, bool) {
+	v, ok := bd.m[string(key)]
+	if !ok {
+		return nil, ok
+	}
+	return v, ok
+}
+
+func (bd *bcastDict) blkLen(key []byte) int {
+	return len(bd.m[string(key)].blks)
+}
+
+func (bd *bcastDict) store(key []byte, d *blksInfo) {
+	bd.m[string(key)] = d
+}
+
+// ConsistentBCast tracks recent information about the
+// blocks and tickets received at different epochs
+type ConsistentBCast struct {
+	lk         sync.RWMutex
+	delay      time.Duration
+	m          map[abi.ChainEpoch]*bcastDict
+	lastDeepGc abi.ChainEpoch
+}
+
+func newBcastDict() *bcastDict {
+	return &bcastDict{m: make(map[string]*blksInfo)}
+}
+
+func BCastKey(bh *types.BlockHeader) []byte {
+	return bh.Ticket.VRFProof
+}
+
+func NewConsistentBCast(delay time.Duration) *ConsistentBCast {
+	return &ConsistentBCast{
+		delay: delay,
+		m:     make(map[abi.ChainEpoch]*bcastDict),
+	}
+}
+
+func cidExists(cids []cid.Cid, c cid.Cid) bool {
+	for _, v := range cids {
+		if v == c {
+			return true
+		}
+	}
+	return false
+}
+
+func (bInfo *blksInfo) eqErr() error {
+	bInfo.cancel()
+	return xerrors.Errorf("different blocks with the same ticket already seen")
+}
+
+func (cb *ConsistentBCast) Len() int {
+	cb.lk.RLock()
+	defer cb.lk.RUnlock()
+	return len(cb.m)
+}
+
+// RcvBlock is called every time a new block is received through the network.
+//
+// This function keeps track of all the blocks with a specific VRFProof received
+// for the same height. Every time a new block with a VRFProof not seen at certain
+// height is received, a new timer is triggered to wait for the delay time determined by
+// the consistent broadcast before informing the syncer. During this time, if a new
+// block with the same VRFProof for that height is received, it means a miner is
+// trying to equivocate, and both blocks are discarded.
+//
+// The delay time should be set to a value high enough to allow any block sent for
+// certain epoch to be propagated to a large amount of miners in the network.
+func (cb *ConsistentBCast) RcvBlock(ctx context.Context, blk *types.BlockMsg) {
+	cb.lk.Lock()
+	defer cb.lk.Unlock()
+	bcastDict, ok := cb.m[blk.Header.Height]
+	if !ok {
+		bcastDict = newBcastDict()
+		cb.m[blk.Header.Height] = bcastDict
+	}
+
+	key := BCastKey(blk.Header)
+	blkCid := blk.Cid()
+
+	bInfo, ok := bcastDict.load(key)
+	if ok {
+		if len(bInfo.blks) > 1 {
+			log.Errorf("equivocation detected for height %d: %s", blk.Header.Height, bInfo.eqErr())
+			return
+		}
+
+		if !cidExists(bInfo.blks, blkCid) {
+			bcastDict.store(key, &blksInfo{bInfo.ctx, bInfo.cancel, append(bInfo.blks, blkCid)})
+			// By calling bInfo.eqErr() inside this log we cancel the context for all blocks waiting for
+			// the epoch-ticket combination making them to fail and not be sent to the syncer, as
+			// a potential equivocation is detected.
+			log.Errorf("equivocation detected for height %d: %s", blk.Header.Height, bInfo.eqErr())
+			return
+		}
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, cb.delay)
+	bcastDict.store(key, &blksInfo{ctx, cancel, []cid.Cid{blkCid}})
+}
+
+// WaitForDelivery is called before informing the syncer about a new block
+// to check if the consistent broadcast delay triggered or if the block should
+// be held off for a bit more time.
+func (cb *ConsistentBCast) WaitForDelivery(bh *types.BlockHeader) error {
+	cb.lk.RLock()
+	bcastDict := cb.m[bh.Height]
+	key := BCastKey(bh)
+	bInfo, ok := bcastDict.load(key)
+	cb.lk.RUnlock()
+	if !ok {
+		return xerrors.Errorf("something went wrong, unknown block with Epoch + VRFProof (cid=%s) in consistent broadcast storage", key)
+	}
+
+	// Wait for the timeout
+	<-bInfo.ctx.Done()
+	if bcastDict.blkLen(key) > 1 {
+		return xerrors.Errorf("equivocation detected for epoch %d. Two blocks being broadcast with same VRFProof", bh.Height)
+	}
+	return nil
+}
+
+// GarbageCollect cleans the consistent broadcast cache periodically.
+//
+// A light garbage collection is triggered before every block delivery
+// while a deeper one is triggered once every GcDeepCheck to ensure
+// that nothing was left behind.
+func (cb *ConsistentBCast) GarbageCollect(currEpoch abi.ChainEpoch) {
+	cb.lk.Lock()
+	defer cb.lk.Unlock()
+
+	// perform a deeper sanity check every now and then
+	gcRange := GcSanityCheck
+	if cb.lastDeepGc+GcDeepInterval > currEpoch {
+		gcRange = GcDeepCheck
+		cb.lastDeepGc = currEpoch
+	}
+
+	// keep currEpoch-gcRange and delete a few more in the past
+	// as a sanity-check
+	// Garbage collection is triggered before block delivery,
+	// and we use the sanity-check in case there were a few rounds
+	// without delivery, and the garbage collection wasn't triggered
+	// for a few epochs.
+	for i := 0; i < gcRange; i++ {
+		if currEpoch > GcLookback {
+			delete(cb.m, currEpoch-abi.ChainEpoch(GcLookback+i))
+		}
+	}
+}

chain/sub/bcast/consistent_test.go

@@ -0,0 +1,223 @@
+package bcast_test
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	mrand "math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/sub/bcast"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+const TEST_DELAY = 1 * time.Second
+
+func TestSimpleDelivery(t *testing.T) {
+	cb := bcast.NewConsistentBCast(TEST_DELAY)
+	// Check that we wait for delivery.
+	start := time.Now()
+	testSimpleDelivery(t, cb, 100, 5)
+	since := time.Since(start)
+	require.GreaterOrEqual(t, since, TEST_DELAY)
+}
+
+func testSimpleDelivery(t *testing.T, cb *bcast.ConsistentBCast, epoch abi.ChainEpoch, numBlocks int) {
+	ctx := context.Background()
+
+	wg := new(sync.WaitGroup)
+	errs := make([]error, 0)
+	wg.Add(numBlocks)
+	for i := 0; i < numBlocks; i++ {
+		go func(i int) {
+			defer wg.Done()
+			// Add a random delay in block reception
+			r := mrand.Intn(200)
+			time.Sleep(time.Duration(r) * time.Millisecond)
+			blk := newBlock(t, epoch, randomProof(t), []byte("test"+strconv.Itoa(i)))
+			cb.RcvBlock(ctx, blk)
+			err := cb.WaitForDelivery(blk.Header)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	for _, v := range errs {
+		t.Fatalf("error in delivery: %s", v)
+	}
+}
+
+func TestSeveralEpochs(t *testing.T) {
+	cb := bcast.NewConsistentBCast(TEST_DELAY)
+	numEpochs := 6
+	wg := new(sync.WaitGroup)
+	wg.Add(numEpochs)
+	for i := 0; i < numEpochs; i++ {
+		go func(i int) {
+			defer wg.Done()
+			// Add a random delay between epochs
+			r := mrand.Intn(500)
+			time.Sleep(time.Duration(i)*TEST_DELAY + time.Duration(r)*time.Millisecond)
+			rNumBlocks := mrand.Intn(5)
+			flip, err := flipCoin(0.7)
+			require.NoError(t, err)
+			t.Logf("Running epoch %d with %d with equivocation=%v", i, rNumBlocks, !flip)
+			if flip {
+				testSimpleDelivery(t, cb, abi.ChainEpoch(i), rNumBlocks)
+			} else {
+				testEquivocation(t, cb, abi.ChainEpoch(i), rNumBlocks)
+			}
+			cb.GarbageCollect(abi.ChainEpoch(i))
+		}(i)
+	}
+	wg.Wait()
+	require.Equal(t, cb.Len(), numEpochs)
+}
+
+// bias is expected to be 0-1
+func flipCoin(bias float32) (bool, error) {
+	if bias > 1 || bias < 0 {
+		return false, fmt.Errorf("wrong bias. expected (0,1)")
+	}
+	r := mrand.Intn(100)
+	return r < int(bias*100), nil
+}
+
+func testEquivocation(t *testing.T, cb *bcast.ConsistentBCast, epoch abi.ChainEpoch, numBlocks int) {
+	ctx := context.Background()
+
+	wg := new(sync.WaitGroup)
+	errs := make([]error, 0)
+	wg.Add(numBlocks + 1)
+	for i := 0; i < numBlocks; i++ {
+		proof := randomProof(t)
+		// Valid blocks
+		go func(i int, proof []byte) {
+			defer wg.Done()
+			r := mrand.Intn(200)
+			time.Sleep(time.Duration(r) * time.Millisecond)
+			blk := newBlock(t, epoch, proof, []byte("valid"+strconv.Itoa(i)))
+			cb.RcvBlock(ctx, blk)
+			err := cb.WaitForDelivery(blk.Header)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}(i, proof)
+
+		// Equivocation for the last block
+		if i == numBlocks-1 {
+			// Attempting equivocation
+			go func(i int, proof []byte) {
+				defer wg.Done()
+				// Use the same proof and the same epoch
+				blk := newBlock(t, epoch, proof, []byte("invalid"+strconv.Itoa(i)))
+				cb.RcvBlock(ctx, blk)
+				err := cb.WaitForDelivery(blk.Header)
+				// Equivocation detected
+				require.Error(t, err)
+			}(i, proof)
+		}
+	}
+	wg.Wait()
+
+	// The equivocated block arrived too late, so
+	// we delivered all the valid blocks.
+	require.Len(t, errs, 1)
+}
+
+func TestEquivocation(t *testing.T) {
+	cb := bcast.NewConsistentBCast(TEST_DELAY)
+	testEquivocation(t, cb, 100, 5)
+}
+
+func TestFailedEquivocation(t *testing.T) {
+	cb := bcast.NewConsistentBCast(TEST_DELAY)
+	ctx := context.Background()
+	numBlocks := 5
+
+	wg := new(sync.WaitGroup)
+	errs := make([]error, 0)
+	wg.Add(numBlocks + 1)
+	for i := 0; i < numBlocks; i++ {
+		proof := randomProof(t)
+		// Valid blocks
+		go func(i int, proof []byte) {
+			defer wg.Done()
+			r := mrand.Intn(200)
+			time.Sleep(time.Duration(r) * time.Millisecond)
+			blk := newBlock(t, 100, proof, []byte("valid"+strconv.Itoa(i)))
+			cb.RcvBlock(ctx, blk)
+			err := cb.WaitForDelivery(blk.Header)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}(i, proof)
+
+		// Equivocation for the last block
+		if i == numBlocks-1 {
+			// Attempting equivocation
+			go func(i int, proof []byte) {
+				defer wg.Done()
+				// The equivocated block arrives late
+				time.Sleep(2 * TEST_DELAY)
+				// Use the same proof and the same epoch
+				blk := newBlock(t, 100, proof, []byte("invalid"+strconv.Itoa(i)))
+				cb.RcvBlock(ctx, blk)
+				err := cb.WaitForDelivery(blk.Header)
+				// Equivocation detected
+				require.Error(t, err)
+			}(i, proof)
+		}
+	}
+	wg.Wait()
+
+	// The equivocated block arrived too late, so
+	// we delivered all the valid blocks.
+	require.Len(t, errs, 0)
+}
+
+func randomProof(t *testing.T) []byte {
+	proof := make([]byte, 10)
+	_, err := rand.Read(proof)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return proof
+}
+
+func newBlock(t *testing.T, epoch abi.ChainEpoch, proof []byte, mCidSeed []byte) *types.BlockMsg {
+	h, err := multihash.Sum(mCidSeed, multihash.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testCid := cid.NewCidV0(h)
+	addr, err := address.NewIDAddress(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bh := &types.BlockHeader{
+		Miner:                 addr,
+		ParentStateRoot:       testCid,
+		ParentMessageReceipts: testCid,
+		Ticket: &types.Ticket{
+			VRFProof: proof,
+		},
+		Height:   epoch,
+		Messages: testCid,
+	}
+	return &types.BlockMsg{
+		Header: bh,
+	}
+}

chain/sub/incoming.go

@@ -27,6 +27,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/sub/bcast"
 	"github.com/filecoin-project/lotus/chain/sub/ratelimit"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/metrics"
@@ -43,10 +44,11 @@ var msgCidPrefix = cid.Prefix{
 	MhLength: 32,
 }
 
-func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) {
+func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, self peer.ID, s *chain.Syncer, bs bserv.BlockService, cmgr connmgr.ConnManager) {
 	// Timeout after (block time + propagation delay). This is useless at
 	// this point.
 	timeout := time.Duration(build.BlockDelaySecs+build.PropagationDelaySecs) * time.Second
+	cb := bcast.NewConsistentBCast(build.CBDeliveryDelay)
 
 	for {
 		msg, err := bsub.Next(ctx)
@@ -67,6 +69,9 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 
 		src := msg.GetFrom()
 
+		// Notify consistent broadcast about a new block
+		cb.RcvBlock(ctx, blk)
+
 		go func() {
 			ctx, cancel := context.WithTimeout(ctx, timeout)
 			defer cancel()
@@ -102,6 +107,20 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 				log.Warnw("received block with large delay from miner", "block", blk.Cid(), "delay", delay, "miner", blk.Header.Miner)
 			}
 
+			// When we propose a new block ourselves, the proposed block also gets here through SyncSubmitBlock.
+			// If we are the block proposers we don't need to wait for delivery, we know the blocks are
+			// honest.
+			if src != self {
+				log.Debugf("Waiting for consistent broadcast of block in height: %v", blk.Header.Height)
+				if err := cb.WaitForDelivery(blk.Header); err != nil {
+					log.Errorf("not informing syncer about new block, potential equivocation detected (cid: %s, source: %s): %s; ", blk.Header.Cid(), src, err)
+					return
+				}
+			}
+			// Garbage collect the broadcast state
+			cb.GarbageCollect(blk.Header.Height)
+			log.Debugf("Block in height %v delivered successfully (cid=%s)", blk.Header.Height, blk.Cid())
+
 			if s.InformNewBlock(msg.ReceivedFrom, &types.FullBlock{
 				Header:        blk.Header,
 				BlsMessages:   bmsgs,

chain/sync_test.go

@@ -45,6 +45,11 @@ func init() {
 	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
 	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
 	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
+
+	// these tests assume really fast block times. disabling
+	// the consistent broadcast delay to avoid them from adding
+	// an unnecessary overhead.
+	build.CBDeliveryDelay = 2 * time.Millisecond
 }
 
 const source = 0

chain/vm/execution.go

@@ -0,0 +1,192 @@
+package vm
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/ipfs/go-cid"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/metrics"
+)
+
+const (
+	// DefaultAvailableExecutionLanes is the number of available execution lanes; it is the bound of
+	// concurrent active executions.
+	// This is the default value in filecoin-ffi
+	DefaultAvailableExecutionLanes = 4
+	// DefaultPriorityExecutionLanes is the number of reserved execution lanes for priority computations.
+	// This is purely userspace, but we believe it is a reasonable default, even with more available
+	// lanes.
+	DefaultPriorityExecutionLanes = 2
+)
+
+// the execution environment; see below for definition, methods, and initialization
+var execution *executionEnv
+
+// implementation of vm executor with simple sanity check preventing use after free.
+type vmExecutor struct {
+	vmi  Interface
+	lane ExecutionLane
+}
+
+var _ Interface = (*vmExecutor)(nil)
+
+func newVMExecutor(vmi Interface, lane ExecutionLane) Interface {
+	return &vmExecutor{vmi: vmi, lane: lane}
+}
+
+func (e *vmExecutor) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, error) {
+	token := execution.getToken(e.lane)
+	defer token.Done()
+
+	return e.vmi.ApplyMessage(ctx, cmsg)
+}
+
+func (e *vmExecutor) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) {
+	token := execution.getToken(e.lane)
+	defer token.Done()
+
+	return e.vmi.ApplyImplicitMessage(ctx, msg)
+}
+
+func (e *vmExecutor) Flush(ctx context.Context) (cid.Cid, error) {
+	return e.vmi.Flush(ctx)
+}
+
+type executionToken struct {
+	lane     ExecutionLane
+	reserved int
+}
+
+func (token *executionToken) Done() {
+	execution.putToken(token)
+}
+
+type executionEnv struct {
+	mx   *sync.Mutex
+	cond *sync.Cond
+
+	// available executors
+	available int
+	// reserved executors
+	reserved int
+}
+
+func (e *executionEnv) getToken(lane ExecutionLane) *executionToken {
+	metricsUp(metrics.VMExecutionWaiting, lane)
+	defer metricsDown(metrics.VMExecutionWaiting, lane)
+
+	e.mx.Lock()
+	defer e.mx.Unlock()
+
+	switch lane {
+	case ExecutionLaneDefault:
+		for e.available <= e.reserved {
+			e.cond.Wait()
+		}
+
+		e.available--
+
+		metricsUp(metrics.VMExecutionRunning, lane)
+		return &executionToken{lane: lane, reserved: 0}
+
+	case ExecutionLanePriority:
+		for e.available == 0 {
+			e.cond.Wait()
+		}
+
+		e.available--
+
+		reserving := 0
+		if e.reserved > 0 {
+			e.reserved--
+			reserving = 1
+		}
+
+		metricsUp(metrics.VMExecutionRunning, lane)
+		return &executionToken{lane: lane, reserved: reserving}
+
+	default:
+		// already checked at interface boundary in NewVM, so this is appropriate
+		panic("bogus execution lane")
+	}
+}
+
+func (e *executionEnv) putToken(token *executionToken) {
+	e.mx.Lock()
+	defer e.mx.Unlock()
+
+	e.available++
+	e.reserved += token.reserved
+
+	// Note: Signal is unsound, because a priority token could wake up a non-priority
+	// goroutnie and lead to deadlock. So Broadcast it must be.
+	e.cond.Broadcast()
+
+	metricsDown(metrics.VMExecutionRunning, token.lane)
+}
+
+func metricsUp(metric *stats.Int64Measure, lane ExecutionLane) {
+	metricsAdjust(metric, lane, 1)
+}
+
+func metricsDown(metric *stats.Int64Measure, lane ExecutionLane) {
+	metricsAdjust(metric, lane, -1)
+}
+
+func metricsAdjust(metric *stats.Int64Measure, lane ExecutionLane, delta int) {
+	laneName := "default"
+	if lane > ExecutionLaneDefault {
+		laneName = "priority"
+	}
+
+	ctx, _ := tag.New(
+		context.Background(),
+		tag.Upsert(metrics.ExecutionLane, laneName),
+	)
+	stats.Record(ctx, metric.M(int64(delta)))
+}
+
+func init() {
+	var err error
+
+	available := DefaultAvailableExecutionLanes
+	if concurrency := os.Getenv("LOTUS_FVM_CONCURRENCY"); concurrency != "" {
+		available, err = strconv.Atoi(concurrency)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	priority := DefaultPriorityExecutionLanes
+	if reserved := os.Getenv("LOTUS_FVM_CONCURRENCY_RESERVED"); reserved != "" {
+		priority, err = strconv.Atoi(reserved)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// some sanity checks
+	if available < 2 {
+		panic("insufficient execution concurrency")
+	}
+
+	if available <= priority {
+		panic("insufficient default execution concurrency")
+	}
+
+	mx := &sync.Mutex{}
+	cond := sync.NewCond(mx)
+
+	execution = &executionEnv{
+		mx:        mx,
+		cond:      cond,
+		available: available,
+		reserved:  priority,
+	}
+}

chain/vm/vm.go

@@ -250,6 +250,8 @@ type VMOpts struct {
 	Tracing        bool
 	// ReturnEvents decodes and returns emitted events.
 	ReturnEvents bool
+	// ExecutionLane specifies the execution priority of the created vm
+	ExecutionLane ExecutionLane
 }
 
 func NewLegacyVM(ctx context.Context, opts *VMOpts) (*LegacyVM, error) {

chain/vm/vmi.go

@@ -2,6 +2,7 @@ package vm
 
 import (
 	"context"
+	"fmt"
 	"os"
 
 	cid "github.com/ipfs/go-cid"
@@ -17,6 +18,15 @@ var (
 	StatApplied uint64
 )
 
+type ExecutionLane int
+
+const (
+	// ExecutionLaneDefault signifies a default, non prioritized execution lane.
+	ExecutionLaneDefault ExecutionLane = iota
+	// ExecutionLanePriority signifies a prioritized execution lane with reserved resources.
+	ExecutionLanePriority
+)
+
 type Interface interface {
 	// Applies the given message onto the VM's current state, returning the result of the execution
 	ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, error)
@@ -33,7 +43,7 @@ type Interface interface {
 // Message failures, unexpected terminations,gas costs, etc. should all be ignored.
 var useFvmDebug = os.Getenv("LOTUS_FVM_DEVELOPER_DEBUG") == "1"
 
-func NewVM(ctx context.Context, opts *VMOpts) (Interface, error) {
+func makeVM(ctx context.Context, opts *VMOpts) (Interface, error) {
 	if opts.NetworkVersion >= network.Version16 {
 		if useFvmDebug {
 			return NewDualExecutionFVM(ctx, opts)
@@ -43,3 +53,18 @@ func NewVM(ctx context.Context, opts *VMOpts) (Interface, error) {
 
 	return NewLegacyVM(ctx, opts)
 }
+
+func NewVM(ctx context.Context, opts *VMOpts) (Interface, error) {
+	switch opts.ExecutionLane {
+	case ExecutionLaneDefault, ExecutionLanePriority:
+	default:
+		return nil, fmt.Errorf("invalid execution lane: %d", opts.ExecutionLane)
+	}
+
+	vmi, err := makeVM(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return newVMExecutor(vmi, opts.ExecutionLane), nil
+}

cli/net.go

@@ -367,7 +367,7 @@ func AddrInfoFromArg(ctx context.Context, cctx *cli.Context) ([]peer.AddrInfo, e
 		pis = append(pis, pi)
 	}
 
-	return pis, err
+	return pis, nil
 }
 
 var NetId = &cli.Command{

cli/state.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"html/template"
 	"io"
-	"io/ioutil"
 	"os"
 	"reflect"
 	"sort"
@@ -1090,7 +1089,7 @@ var StateComputeStateCmd = &cli.Command{
 
 		var stout *lapi.ComputeStateOutput
 		if csofile := cctx.String("compute-state-output"); csofile != "" {
-			data, err := ioutil.ReadFile(csofile)
+			data, err := os.ReadFile(csofile)
 			if err != nil {
 				return err
 			}
@@ -1269,7 +1268,7 @@ var compStateMsg = `
  {{end}}
 
  {{if ne .MsgRct.ExitCode 0}}
-  <div class="error">Error: <pre>{{.Error}}</pre></div>
+  <div class="error">Exit: <pre>{{.MsgRct.ExitCode}}</pre></div>
  {{end}}
 
 <details>
@@ -1373,7 +1372,14 @@ func isVerySlow(t time.Duration) bool {
 }
 
 func JsonParams(code cid.Cid, method abi.MethodNum, params []byte) (string, error) {
-	p, err := stmgr.GetParamType(consensus.NewActorRegistry(), code, method) // todo use api for correct actor registry
+	ar := consensus.NewActorRegistry()
+
+	_, found := ar.Methods[code][method]
+	if !found {
+		return fmt.Sprintf("raw:%x", params), nil
+	}
+
+	p, err := stmgr.GetParamType(ar, code, method) // todo use api for correct actor registry
 	if err != nil {
 		return "", err
 	}

cli/util/retrieval.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -65,7 +64,7 @@ func ClientExportStream(apiAddr string, apiAuth http.Header, eref api.ExportRef,
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		em, err := ioutil.ReadAll(resp.Body)
+		em, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return nil, xerrors.Errorf("reading error body: %w", err)
 		}

cli/wallet.go

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -337,7 +336,7 @@ var walletImport = &cli.Command{
 			inpdata = indata
 
 		} else {
-			fdata, err := ioutil.ReadFile(cctx.Args().First())
+			fdata, err := os.ReadFile(cctx.Args().First())
 			if err != nil {
 				return err
 			}

cmd/lotus-bench/import.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math"
 	"net/http"
 	_ "net/http/pprof"
@@ -36,6 +35,7 @@ import (
 	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -158,7 +158,7 @@ var importBenchCmd = &cli.Command{
 		if rdir := cctx.String("repodir"); rdir != "" {
 			tdir = rdir
 		} else {
-			tmp, err := ioutil.TempDir("", "lotus-import-bench")
+			tmp, err := os.MkdirTemp("", "lotus-import-bench")
 			if err != nil {
 				return err
 			}
@@ -229,7 +229,7 @@ var importBenchCmd = &cli.Command{
 		defer cs.Close() //nolint:errcheck
 
 		// TODO: We need to supply the actual beacon after v14
-		stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil, metadataDs)
+		stm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(verifier), filcns.DefaultUpgradeSchedule(), nil, metadataDs, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}

cmd/lotus-bench/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"math/rand"
 	"os"
@@ -197,7 +196,7 @@ var sealBenchCmd = &cli.Command{
 				return xerrors.Errorf("creating sectorbuilder dir: %w", err)
 			}
 
-			tsdir, err := ioutil.TempDir(sdir, "bench")
+			tsdir, err := os.MkdirTemp(sdir, "bench")
 			if err != nil {
 				return err
 			}
@@ -287,7 +286,7 @@ var sealBenchCmd = &cli.Command{
 			// sectorbuilder directory... we need a better way to handle
 			// this in other cases
 
-			fdata, err := ioutil.ReadFile(filepath.Join(sbdir, "pre-seal-"+maddr.String()+".json"))
+			fdata, err := os.ReadFile(filepath.Join(sbdir, "pre-seal-"+maddr.String()+".json"))
 			if err != nil {
 				return err
 			}
@@ -637,7 +636,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 							return err
 						}
 
-						if err := ioutil.WriteFile(saveC2inp, b, 0664); err != nil {
+						if err := os.WriteFile(saveC2inp, b, 0664); err != nil {
 							log.Warnf("%+v", err)
 						}
 					}
@@ -751,7 +750,7 @@ var proveCmd = &cli.Command{
 			return xerrors.Errorf("Usage: lotus-bench prove [input.json]")
 		}
 
-		inb, err := ioutil.ReadFile(c.Args().First())
+		inb, err := os.ReadFile(c.Args().First())
 		if err != nil {
 			return xerrors.Errorf("reading input file: %w", err)
 		}

cmd/lotus-bench/simple.go

@@ -5,7 +5,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"time"
@@ -444,7 +443,7 @@ var simpleCommit1 = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(cctx.Args().Get(4), b, 0664); err != nil {
+		if err := os.WriteFile(cctx.Args().Get(4), b, 0664); err != nil {
 			log.Warnf("%+v", err)
 		}
 
@@ -478,7 +477,7 @@ var simpleCommit2 = &cli.Command{
 			return xerrors.Errorf("Usage: lotus-bench prove [input.json]")
 		}
 
-		inb, err := ioutil.ReadFile(c.Args().First())
+		inb, err := os.ReadFile(c.Args().First())
 		if err != nil {
 			return xerrors.Errorf("reading input file: %w", err)
 		}
@@ -861,7 +860,7 @@ var simpleProveReplicaUpdate1 = &cli.Command{
 			return xerrors.Errorf("json marshal vanilla proofs: %w", err)
 		}
 
-		if err := ioutil.WriteFile(cctx.Args().Get(7), vpjb, 0666); err != nil {
+		if err := os.WriteFile(cctx.Args().Get(7), vpjb, 0666); err != nil {
 			return xerrors.Errorf("writing vanilla proofs file: %w", err)
 		}
 
@@ -934,7 +933,7 @@ var simpleProveReplicaUpdate2 = &cli.Command{
 			return xerrors.Errorf("parse commr: %w", err)
 		}
 
-		vpb, err := ioutil.ReadFile(cctx.Args().Get(3))
+		vpb, err := os.ReadFile(cctx.Args().Get(3))
 		if err != nil {
 			return xerrors.Errorf("reading valilla proof file: %w", err)
 		}

cmd/lotus-fountain/recaptcha.go

@@ -6,7 +6,7 @@ package main
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -63,7 +63,7 @@ func VerifyToken(token, remoteIP string) (Response, error) {
 		return resp, err
 	}
 
-	b, err := ioutil.ReadAll(r.Body)
+	b, err := io.ReadAll(r.Body)
 	_ = r.Body.Close() // close immediately after reading finished
 	if err != nil {
 		return resp, err

cmd/lotus-miner/info.go

@@ -325,14 +325,12 @@ func handleMiningInfo(ctx context.Context, cctx *cli.Context, fullapi v1api.Full
 		}
 	}
 
-	{
-		fmt.Println()
-
-		ws, err := nodeApi.WorkerStats(ctx)
-		if err != nil {
-			return xerrors.Errorf("getting worker stats: %w", err)
-		}
+	fmt.Println()
 
+	ws, err := nodeApi.WorkerStats(ctx)
+	if err != nil {
+		fmt.Printf("ERROR: getting worker stats: %s\n", err)
+	} else {
 		workersByType := map[string]int{
 			sealtasks.WorkerSealing:     0,
 			sealtasks.WorkerWindowPoSt:  0,

cmd/lotus-miner/init.go

@@ -7,7 +7,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -246,7 +245,7 @@ var initCmd = &cli.Command{
 					return xerrors.Errorf("marshaling storage config: %w", err)
 				}
 
-				if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil {
+				if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil {
 					return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err)
 				}
 
@@ -292,7 +291,7 @@ func migratePreSealMeta(ctx context.Context, api v1api.FullNode, metadata string
 		return xerrors.Errorf("expanding preseal dir: %w", err)
 	}
 
-	b, err := ioutil.ReadFile(metadata)
+	b, err := os.ReadFile(metadata)
 	if err != nil {
 		return xerrors.Errorf("reading preseal metadata: %w", err)
 	}

cmd/lotus-miner/init_restore.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"io/ioutil"
 	"os"
 
 	"github.com/docker/go-units"
@@ -59,7 +58,7 @@ var restoreCmd = &cli.Command{
 				return xerrors.Errorf("expanding storage config path: %w", err)
 			}
 
-			cfb, err := ioutil.ReadFile(cf)
+			cfb, err := os.ReadFile(cf)
 			if err != nil {
 				return xerrors.Errorf("reading storage config: %w", err)
 			}

cmd/lotus-miner/proving.go

@@ -658,6 +658,10 @@ It will not send any messages to the chain.`,
 		for _, i := range res {
 			var postParam SubmitWindowedPoStParams
 			postParam.Deadline = i.Deadline
+
+			// Initialize the postParam.Partitions slice with the same length as i.Partitions
+			postParam.Partitions = make([]PoStPartition, len(i.Partitions))
+
 			for id, part := range i.Partitions {
 				postParam.Partitions[id].Index = part.Index
 				count, err := part.Skipped.Count()

cmd/lotus-miner/sectors.go

@@ -1160,8 +1160,9 @@ var sectorsExtendCmd = &cli.Command{
 				}
 
 				sectorsInDecl := int(sectorsWithoutClaimsCount) + len(sectorsWithClaims)
+				scount += sectorsInDecl
 
-				if scount+sectorsInDecl > addrSectors || len(p.Extensions) >= declMax {
+				if scount > addrSectors || len(p.Extensions) >= declMax {
 					params = append(params, p)
 					p = miner.ExtendSectorExpiration2Params{}
 					scount = sectorsInDecl
@@ -1230,7 +1231,7 @@ var sectorsExtendCmd = &cli.Command{
 			smsg, err := fullApi.MpoolPushMessage(ctx, &types.Message{
 				From:   mi.Worker,
 				To:     maddr,
-				Method: builtin.MethodsMiner.ExtendSectorExpiration,
+				Method: builtin.MethodsMiner.ExtendSectorExpiration2,
 				Value:  big.Zero(),
 				Params: sp,
 			}, spec)
@@ -1391,6 +1392,12 @@ var sectorsRemoveCmd = &cli.Command{
 			return xerrors.Errorf("could not parse sector number: %w", err)
 		}
 
+		// Check if the sector exists
+		_, err = minerAPI.SectorsStatus(ctx, abi.SectorNumber(id), false)
+		if err != nil {
+			return xerrors.Errorf("sectorID %d has not been created yet: %w", id, err)
+		}
+
 		return minerAPI.SectorRemove(ctx, abi.SectorNumber(id))
 	},
 }

cmd/lotus-miner/storage.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/bits"
 	"os"
 	"path/filepath"
@@ -167,7 +166,7 @@ over time
 				return xerrors.Errorf("marshaling storage config: %w", err)
 			}
 
-			if err := ioutil.WriteFile(filepath.Join(p, metaFile), b, 0644); err != nil {
+			if err := os.WriteFile(filepath.Join(p, metaFile), b, 0644); err != nil {
 				return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(p, metaFile), err)
 			}
 		}

cmd/lotus-pcr/main.go

@@ -7,7 +7,6 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
@@ -759,7 +758,7 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet
 		return nil, err
 	}
 
-	w := ioutil.Discard
+	w := io.Discard
 	if len(output) != 0 {
 		f, err := os.Create(output)
 		if err != nil {
@@ -1299,7 +1298,7 @@ func loadChainEpoch(fn string) (abi.ChainEpoch, error) {
 		err = f.Close()
 	}()
 
-	raw, err := ioutil.ReadAll(f)
+	raw, err := io.ReadAll(f)
 	if err != nil {
 		return 0, err
 	}

cmd/lotus-seed/genesis.go

@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -80,7 +79,7 @@ var genesisNewCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, genb, 0644); err != nil {
+		if err := os.WriteFile(genf, genb, 0644); err != nil {
 			return err
 		}
 
@@ -103,7 +102,7 @@ var genesisAddMinerCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		genb, err := ioutil.ReadFile(genf)
+		genb, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -117,7 +116,7 @@ var genesisAddMinerCmd = &cli.Command{
 			return xerrors.Errorf("expand preseal file path: %w", err)
 		}
 		miners := map[string]genesis.Miner{}
-		minb, err := ioutil.ReadFile(minf)
+		minb, err := os.ReadFile(minf)
 		if err != nil {
 			return xerrors.Errorf("read preseal file: %w", err)
 		}
@@ -156,7 +155,7 @@ var genesisAddMinerCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, genb, 0644); err != nil {
+		if err := os.WriteFile(genf, genb, 0644); err != nil {
 			return err
 		}
 
@@ -196,7 +195,7 @@ var genesisAddMsigsCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		b, err := ioutil.ReadFile(genf)
+		b, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -237,7 +236,7 @@ var genesisAddMsigsCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+		if err := os.WriteFile(genf, b, 0644); err != nil {
 			return err
 		}
 		return nil
@@ -339,7 +338,7 @@ var genesisSetVRKCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		b, err := ioutil.ReadFile(genf)
+		b, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -404,7 +403,7 @@ var genesisSetVRKCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+		if err := os.WriteFile(genf, b, 0644); err != nil {
 			return err
 		}
 		return nil
@@ -435,7 +434,7 @@ var genesisSetRemainderCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		b, err := ioutil.ReadFile(genf)
+		b, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -500,7 +499,7 @@ var genesisSetRemainderCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+		if err := os.WriteFile(genf, b, 0644); err != nil {
 			return err
 		}
 		return nil
@@ -529,7 +528,7 @@ var genesisSetActorVersionCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		b, err := ioutil.ReadFile(genf)
+		b, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -550,7 +549,7 @@ var genesisSetActorVersionCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+		if err := os.WriteFile(genf, b, 0644); err != nil {
 			return err
 		}
 		return nil
@@ -607,7 +606,7 @@ var genesisSetVRKSignersCmd = &cli.Command{
 		}
 
 		var template genesis.Template
-		b, err := ioutil.ReadFile(genf)
+		b, err := os.ReadFile(genf)
 		if err != nil {
 			return xerrors.Errorf("read genesis template: %w", err)
 		}
@@ -662,7 +661,7 @@ var genesisSetVRKSignersCmd = &cli.Command{
 			return err
 		}
 
-		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+		if err := os.WriteFile(genf, b, 0644); err != nil {
 			return err
 		}
 		return nil

cmd/lotus-seed/main.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 
 	"github.com/docker/go-units"
@@ -114,7 +113,7 @@ var preSealCmd = &cli.Command{
 		var k *types.KeyInfo
 		if c.String("key") != "" {
 			k = new(types.KeyInfo)
-			kh, err := ioutil.ReadFile(c.String("key"))
+			kh, err := os.ReadFile(c.String("key"))
 			if err != nil {
 				return err
 			}

cmd/lotus-seed/seed/seed.go

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -135,7 +134,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
 			return nil, nil, xerrors.Errorf("marshaling storage config: %w", err)
 		}
 
-		if err := ioutil.WriteFile(filepath.Join(sbroot, "sectorstore.json"), b, 0644); err != nil {
+		if err := os.WriteFile(filepath.Join(sbroot, "sectorstore.json"), b, 0644); err != nil {
 			return nil, nil, xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(sbroot, "storage.json"), err)
 		}
 	}
@@ -228,7 +227,7 @@ func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner,
 
 	log.Infof("Writing preseal manifest to %s", filepath.Join(sbroot, "pre-seal-"+maddr.String()+".json"))
 
-	if err := ioutil.WriteFile(filepath.Join(sbroot, "pre-seal-"+maddr.String()+".json"), out, 0664); err != nil {
+	if err := os.WriteFile(filepath.Join(sbroot, "pre-seal-"+maddr.String()+".json"), out, 0664); err != nil {
 		return err
 	}
 
@@ -239,7 +238,7 @@ func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.Miner,
 		}
 
 		// TODO: allow providing key
-		if err := ioutil.WriteFile(filepath.Join(sbroot, "pre-seal-"+maddr.String()+".key"), []byte(hex.EncodeToString(b)), 0664); err != nil {
+		if err := os.WriteFile(filepath.Join(sbroot, "pre-seal-"+maddr.String()+".key"), []byte(hex.EncodeToString(b)), 0664); err != nil {
 			return err
 		}
 	}

cmd/lotus-shed/balances.go

@@ -35,6 +35,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
 	"github.com/filecoin-project/lotus/chain/gen/genesis"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -513,7 +514,7 @@ var chainBalanceStateCmd = &cli.Command{
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)
 
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}
@@ -737,7 +738,7 @@ var chainPledgeCmd = &cli.Command{
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)
 
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/base16.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -30,7 +29,7 @@ var base16Cmd = &cli.Command{
 			input = strings.NewReader(cctx.Args().First())
 		}
 
-		bytes, err := ioutil.ReadAll(input)
+		bytes, err := io.ReadAll(input)
 		if err != nil {
 			return nil
 		}

cmd/lotus-shed/base32.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -30,7 +29,7 @@ var base32Cmd = &cli.Command{
 			input = strings.NewReader(cctx.Args().First())
 		}
 
-		bytes, err := ioutil.ReadAll(input)
+		bytes, err := io.ReadAll(input)
 		if err != nil {
 			return nil
 		}

cmd/lotus-shed/base64.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -38,7 +37,7 @@ var base64Cmd = &cli.Command{
 			input = strings.NewReader(cctx.Args().First())
 		}
 
-		bytes, err := ioutil.ReadAll(input)
+		bytes, err := io.ReadAll(input)
 		if err != nil {
 			return nil
 		}

cmd/lotus-shed/bitfield.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	"github.com/urfave/cli/v2"
@@ -318,7 +317,7 @@ func decodeToByte(cctx *cli.Context, i int) ([]byte, error) {
 		if i > 0 {
 			return nil, xerrors.Errorf("need more than %d args", i)
 		}
-		r, err := ioutil.ReadAll(os.Stdin)
+		r, err := io.ReadAll(os.Stdin)
 		if err != nil {
 			return nil, err
 		}

cmd/lotus-shed/chainwatch.go

@@ -0,0 +1,383 @@
+package main
+
+import (
+	"container/list"
+	"context"
+	"database/sql"
+	"fmt"
+	"hash/crc32"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+)
+
+var chainwatchCmd = &cli.Command{
+	Name:  "chainwatch",
+	Usage: "lotus chainwatch",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "db",
+			EnvVars: []string{"CHAINWATCH_DB"},
+			Value:   "./chainwatch.db",
+		},
+	},
+	Subcommands: []*cli.Command{
+		chainwatchRunCmd,
+		chainwatchDotCmd,
+	},
+}
+
+var chainwatchDotCmd = &cli.Command{
+	Name:      "dot",
+	Usage:     "generate dot graphs",
+	ArgsUsage: "<minHeight> <toseeHeight>",
+	Action: func(cctx *cli.Context) error {
+		st, err := cwOpenStorage(cctx.String("db"))
+		if err != nil {
+			return err
+		}
+
+		minH, err := strconv.ParseInt(cctx.Args().Get(0), 10, 32)
+		if err != nil {
+			return err
+		}
+		tosee, err := strconv.ParseInt(cctx.Args().Get(1), 10, 32)
+		if err != nil {
+			return err
+		}
+		maxH := minH + tosee
+
+		res, err := st.db.Query(`select block, parent, b.miner, b.height, p.height from block_parents
+    inner join blocks b on block_parents.block = b.cid
+    inner join blocks p on block_parents.parent = p.cid
+where b.height > ? and b.height < ?`, minH, maxH)
+
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("digraph D {")
+
+		for res.Next() {
+			var block, parent, miner string
+			var height, ph uint64
+			if err := res.Scan(&block, &parent, &miner, &height, &ph); err != nil {
+				return err
+			}
+
+			bc, err := cid.Parse(block)
+			if err != nil {
+				return err
+			}
+
+			has := st.hasBlock(bc)
+
+			col := crc32.Checksum([]byte(miner), crc32.MakeTable(crc32.Castagnoli))&0xc0c0c0c0 + 0x30303030
+
+			hasstr := ""
+			if !has {
+				//col = 0xffffffff
+				hasstr = " UNSYNCED"
+			}
+
+			nulls := height - ph - 1
+			for i := uint64(0); i < nulls; i++ {
+				name := block + "NP" + fmt.Sprint(i)
+
+				fmt.Printf("%s [label = \"NULL:%d\", fillcolor = \"#ffddff\", style=filled, forcelabels=true]\n%s -> %s\n",
+					name, height-nulls+i, name, parent)
+
+				parent = name
+			}
+
+			fmt.Printf("%s [label = \"%s:%d%s\", fillcolor = \"#%06x\", style=filled, forcelabels=true]\n%s -> %s\n", block, miner, height, hasstr, col, block, parent)
+		}
+		if res.Err() != nil {
+			return res.Err()
+		}
+
+		fmt.Println("}")
+
+		return nil
+	},
+}
+
+var chainwatchRunCmd = &cli.Command{
+	Name:  "run",
+	Usage: "Start lotus chainwatch",
+
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := cliutil.GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := cliutil.ReqContext(cctx)
+
+		v, err := api.Version(ctx)
+		if err != nil {
+			return err
+		}
+
+		log.Infof("Remote version: %s", v.Version)
+
+		st, err := cwOpenStorage(cctx.String("db")) // todo flag
+		if err != nil {
+			return err
+		}
+		defer st.close() // nolint:errcheck
+
+		cwRunSyncer(ctx, api, st)
+		go cwSubBlocks(ctx, api, st)
+
+		<-ctx.Done()
+		return nil
+	},
+}
+
+func cwSubBlocks(ctx context.Context, api api.FullNode, st *cwStorage) {
+	sub, err := api.SyncIncomingBlocks(ctx)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	for bh := range sub {
+		err := st.storeHeaders(map[cid.Cid]*types.BlockHeader{
+			bh.Cid(): bh,
+		}, false)
+		if err != nil {
+			log.Error(err)
+		}
+	}
+}
+
+func cwRunSyncer(ctx context.Context, api api.FullNode, st *cwStorage) {
+	notifs, err := api.ChainNotify(ctx)
+	if err != nil {
+		panic(err)
+	}
+	go func() {
+		for notif := range notifs {
+			for _, change := range notif {
+				switch change.Type {
+				case store.HCCurrent:
+					fallthrough
+				case store.HCApply:
+					syncHead(ctx, api, st, change.Val)
+				case store.HCRevert:
+					log.Warnf("revert todo")
+				}
+			}
+		}
+	}()
+}
+
+func syncHead(ctx context.Context, api api.FullNode, st *cwStorage, ts *types.TipSet) {
+	log.Infof("Getting headers / actors")
+
+	toSync := map[cid.Cid]*types.BlockHeader{}
+	toVisit := list.New()
+
+	for _, header := range ts.Blocks() {
+		toVisit.PushBack(header)
+	}
+
+	for toVisit.Len() > 0 {
+		bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader)
+
+		if _, seen := toSync[bh.Cid()]; seen || st.hasBlock(bh.Cid()) {
+			continue
+		}
+
+		toSync[bh.Cid()] = bh
+
+		if len(toSync)%500 == 10 {
+			log.Infof("todo: (%d) %s", len(toSync), bh.Cid())
+		}
+
+		if len(bh.Parents) == 0 {
+			continue
+		}
+
+		if bh.Height <= 530000 {
+			continue
+		}
+
+		pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...))
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+
+		for _, header := range pts.Blocks() {
+			toVisit.PushBack(header)
+		}
+	}
+
+	log.Infof("Syncing %d blocks", len(toSync))
+
+	log.Infof("Persisting headers")
+	if err := st.storeHeaders(toSync, true); err != nil {
+		log.Error(err)
+		return
+	}
+
+	log.Infof("Sync done")
+}
+
+type cwStorage struct {
+	db *sql.DB
+
+	headerLk sync.Mutex
+}
+
+func cwOpenStorage(dbSource string) (*cwStorage, error) {
+	db, err := sql.Open("sqlite3", dbSource)
+	if err != nil {
+		return nil, err
+	}
+
+	st := &cwStorage{db: db}
+
+	return st, st.setup()
+}
+
+func (st *cwStorage) setup() error {
+	tx, err := st.db.Begin()
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(`
+create table if not exists blocks
+(
+	cid text not null
+		constraint blocks_pk
+			primary key,
+	parentWeight numeric not null,
+	parentStateRoot text not null,
+	height int not null,
+	miner text not null
+		constraint blocks_id_address_map_miner_fk
+			references id_address_map (address),
+	timestamp int not null,
+	vrfproof blob
+);
+
+create unique index if not exists block_cid_uindex
+	on blocks (cid);
+
+create table if not exists blocks_synced
+(
+	cid text not null
+		constraint blocks_synced_pk
+			primary key
+		constraint blocks_synced_blocks_cid_fk
+			references blocks,
+	add_ts int not null
+);
+
+create unique index if not exists blocks_synced_cid_uindex
+	on blocks_synced (cid);
+
+create table if not exists block_parents
+(
+	block text not null
+		constraint block_parents_blocks_cid_fk
+			references blocks,
+	parent text not null
+		constraint block_parents_blocks_cid_fk_2
+			references blocks
+);
+
+create unique index if not exists block_parents_block_parent_uindex
+	on block_parents (block, parent);
+
+create unique index if not exists blocks_cid_uindex
+	on blocks (cid);
+`)
+	if err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+
+func (st *cwStorage) hasBlock(bh cid.Cid) bool {
+	var exitsts bool
+	err := st.db.QueryRow(`select exists (select 1 FROM blocks_synced where cid=?)`, bh.String()).Scan(&exitsts)
+	if err != nil {
+		log.Error(err)
+		return false
+	}
+	return exitsts
+}
+
+func (st *cwStorage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) error {
+	st.headerLk.Lock()
+	defer st.headerLk.Unlock()
+
+	tx, err := st.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof) values (?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close() // nolint:errcheck
+	for _, bh := range bhs {
+		if _, err := stmt.Exec(bh.Cid().String(),
+			bh.ParentWeight.String(),
+			bh.ParentStateRoot.String(),
+			bh.Height,
+			bh.Miner.String(),
+			bh.Timestamp,
+			bh.Ticket.VRFProof,
+		); err != nil {
+			return err
+		}
+	}
+
+	stmt2, err := tx.Prepare(`insert into block_parents (block, parent) values (?, ?) on conflict do nothing`)
+	if err != nil {
+		return err
+	}
+	defer stmt2.Close() // nolint:errcheck
+	for _, bh := range bhs {
+		for _, parent := range bh.Parents {
+			if _, err := stmt2.Exec(bh.Cid().String(), parent.String()); err != nil {
+				return err
+			}
+		}
+	}
+
+	if sync {
+		stmt, err := tx.Prepare(`insert into blocks_synced (cid, add_ts) values (?, ?) on conflict do nothing`)
+		if err != nil {
+			return err
+		}
+		defer stmt.Close() // nolint:errcheck
+		now := time.Now().Unix()
+
+		for _, bh := range bhs {
+			if _, err := stmt.Exec(bh.Cid().String(), now); err != nil {
+				return err
+			}
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (st *cwStorage) close() error {
+	return st.db.Close()
+}

cmd/lotus-shed/fevmanalytics.go

@@ -0,0 +1,257 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/net/context"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	evm2 "github.com/filecoin-project/lotus/chain/actors/builtin/evm"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var FevmAnalyticsCmd = &cli.Command{
+	Name:  "evm-analytics",
+	Usage: "Get FEVM related metrics",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+		},
+	},
+	Subcommands: []*cli.Command{
+		FevmBalanceCmd,
+		FevmActorsCmd,
+	},
+}
+
+var FevmBalanceCmd = &cli.Command{
+	Name:      "evm-balance",
+	Usage:     "Balances in eth accounts, evm contracts and placeholders",
+	ArgsUsage: "[state root]",
+
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return xerrors.New("only needs state root")
+		}
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass state root")
+		}
+
+		sroot, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("failed to parse input: %w", err)
+		}
+
+		fsrepo, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		lkrepo, err := fsrepo.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+
+		defer lkrepo.Close() //nolint:errcheck
+
+		path, err := lkrepo.SplitstorePath()
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(path, "hot.badger")
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+
+		opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly())
+		if err != nil {
+			return err
+		}
+
+		bs, err := badgerbs.Open(opts)
+		if err != nil {
+			return err
+		}
+
+		cst := cbor.NewCborStore(bs)
+		st, err := state.LoadStateTree(cst, sroot)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("iterating over all actors")
+		count := 0
+		balanceEvm := abi.NewTokenAmount(0)
+		balanceEthAccount := abi.NewTokenAmount(0)
+		balancePlaceholder := abi.NewTokenAmount(0)
+
+		err = st.ForEach(func(addr address.Address, act *types.Actor) error {
+			if count%200000 == 0 {
+				fmt.Println("processed /n", count)
+			}
+			count++
+
+			if builtin.IsEvmActor(act.Code) {
+				balanceEvm = types.BigAdd(balanceEvm, act.Balance)
+			}
+
+			if builtin.IsEthAccountActor(act.Code) {
+				balanceEthAccount = types.BigAdd(balanceEthAccount, act.Balance)
+			}
+
+			if builtin.IsPlaceholderActor(act.Code) {
+				balancePlaceholder = types.BigAdd(balancePlaceholder, act.Balance)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("balances in Eth contracts: ", balanceEvm)
+		fmt.Println("balances in Eth accounts: ", balanceEthAccount)
+		fmt.Println("balances in placeholder: ", balancePlaceholder)
+		fmt.Println("Total balances: ", big.Add(big.Add(balanceEthAccount, balancePlaceholder), balanceEvm))
+		return nil
+	},
+}
+
+var FevmActorsCmd = &cli.Command{
+	Name:      "evm-actors",
+	Usage:     "actors # in eth accounts, evm contracts and placeholders",
+	ArgsUsage: "[state root]",
+
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return xerrors.New("only needs state root")
+		}
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass state root")
+		}
+
+		sroot, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("failed to parse input: %w", err)
+		}
+
+		fsrepo, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		lkrepo, err := fsrepo.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+
+		defer lkrepo.Close() //nolint:errcheck
+
+		path, err := lkrepo.SplitstorePath()
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(path, "hot.badger")
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+
+		opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly())
+		if err != nil {
+			return err
+		}
+
+		bs, err := badgerbs.Open(opts)
+		if err != nil {
+			return err
+		}
+
+		ctx := context.TODO()
+		cst := cbor.NewCborStore(bs)
+		store := adt.WrapStore(ctx, cst)
+
+		st, err := state.LoadStateTree(cst, sroot)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("iterating over all actors")
+		count := 0
+		EvmCount := 0
+		EthAccountCount := 0
+		PlaceholderCount := 0
+		ea := []cid.Cid{}
+
+		err = st.ForEach(func(addr address.Address, act *types.Actor) error {
+			if count%200000 == 0 {
+				fmt.Println("processed /n", count)
+			}
+			count++
+
+			if builtin.IsEvmActor(act.Code) {
+				EvmCount++
+				e, err := evm2.Load(store, act)
+				if err != nil {
+					return xerrors.Errorf("fail to load evm actor: %w", err)
+				}
+				bcid, err := e.GetBytecodeCID()
+				if err != nil {
+					return err
+				}
+
+				ea = append(ea, bcid)
+			}
+
+			if builtin.IsEthAccountActor(act.Code) {
+				EthAccountCount++
+			}
+
+			if builtin.IsPlaceholderActor(act.Code) {
+				PlaceholderCount++
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		uniquesa := unique(ea)
+		fmt.Println("# of EVM contracts: ", EvmCount)
+		fmt.Println("# of unqiue EVM contracts: ", len(uniquesa))
+		fmt.Println("b# of Eth accounts: ", EthAccountCount)
+		fmt.Println("# of placeholder: ", PlaceholderCount)
+		return nil
+	},
+}
+
+func unique(intSlice []cid.Cid) []cid.Cid {
+	keys := make(map[cid.Cid]bool)
+	list := []cid.Cid{}
+	for _, entry := range intSlice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+	return list
+}

cmd/lotus-shed/fip-0036.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"os"
 	"sort"
 	"strconv"
 
@@ -537,7 +537,7 @@ var finalResultCmd = &cli.Command{
 // Returns voted sorted by votes from earliest to latest
 func getVotesMap(file string) ([]Vote, error) {
 	var votes []Vote
-	vb, err := ioutil.ReadFile(file)
+	vb, err := os.ReadFile(file)
 	if err != nil {
 		return nil, xerrors.Errorf("read vote: %w", err)
 	}

cmd/lotus-shed/gas-estimation.go

@@ -20,6 +20,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/beacon/drand"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -111,7 +112,7 @@ var gasTraceCmd = &cli.Command{
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 		defer cs.Close() //nolint:errcheck
 
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds)
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}
@@ -212,7 +213,7 @@ var replayOfflineCmd = &cli.Command{
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 		defer cs.Close() //nolint:errcheck
 
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds)
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), shd, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}
@@ -240,7 +241,7 @@ var replayOfflineCmd = &cli.Command{
 		}
 
 		tw := tabwriter.NewWriter(os.Stdout, 8, 2, 2, ' ', tabwriter.AlignRight)
-		res, err := sm.CallWithGas(ctx, msg, []types.ChainMsg{}, executionTs)
+		res, err := sm.CallWithGas(ctx, msg, []types.ChainMsg{}, executionTs, true)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/invariants.go

@@ -21,6 +21,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -90,7 +91,7 @@ var invariantsCmd = &cli.Command{
 		cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 		defer cs.Close() //nolint:errcheck
 
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/jwt.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -81,7 +80,7 @@ var jwtTokenCmd = &cli.Command{
 		defer inputFile.Close() //nolint:errcheck
 		input := bufio.NewReader(inputFile)
 
-		encoded, err := ioutil.ReadAll(input)
+		encoded, err := io.ReadAll(input)
 		if err != nil {
 			return err
 		}
@@ -123,7 +122,7 @@ var jwtTokenCmd = &cli.Command{
 			return err
 		}
 
-		return ioutil.WriteFile(cctx.String("output"), token, 0600)
+		return os.WriteFile(cctx.String("output"), token, 0600)
 	},
 }
 
@@ -142,7 +141,7 @@ var jwtNewCmd = &cli.Command{
 
 		keyName := cctx.Args().First()
 
-		sk, err := ioutil.ReadAll(io.LimitReader(rand.Reader, 32))
+		sk, err := io.ReadAll(io.LimitReader(rand.Reader, 32))
 		if err != nil {
 			return err
 		}
@@ -184,6 +183,6 @@ var jwtNewCmd = &cli.Command{
 		}
 
 		filenameToken := fmt.Sprintf("jwt-%s.token", keyName)
-		return ioutil.WriteFile(filenameToken, token, 0600)
+		return os.WriteFile(filenameToken, token, 0600)
 	},
 }

cmd/lotus-shed/keyinfo.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -68,7 +67,7 @@ var keyinfoVerifyCmd = &cli.Command{
 		defer inputFile.Close() //nolint:errcheck
 		input := bufio.NewReader(inputFile)
 
-		keyContent, err := ioutil.ReadAll(input)
+		keyContent, err := io.ReadAll(input)
 		if err != nil {
 			return err
 		}
@@ -162,7 +161,7 @@ var keyinfoImportCmd = &cli.Command{
 			input = bufio.NewReader(inputFile)
 		}
 
-		encoded, err := ioutil.ReadAll(input)
+		encoded, err := io.ReadAll(input)
 		if err != nil {
 			return err
 		}
@@ -274,7 +273,7 @@ var keyinfoInfoCmd = &cli.Command{
 			input = bufio.NewReader(inputFile)
 		}
 
-		encoded, err := ioutil.ReadAll(input)
+		encoded, err := io.ReadAll(input)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/main.go

@@ -26,6 +26,7 @@ func main() {
 		base32Cmd,
 		base16Cmd,
 		bitFieldCmd,
+		chainwatchCmd,
 		cronWcCmd,
 		frozenMinersCmd,
 		dealLabelCmd,
@@ -83,6 +84,8 @@ func main() {
 		invariantsCmd,
 		gasTraceCmd,
 		replayOfflineCmd,
+		msgindexCmd,
+		FevmAnalyticsCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/migrations.go

@@ -41,6 +41,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -123,7 +124,7 @@ var migrationsCmd = &cli.Command{
 		defer cs.Close() //nolint:errcheck
 
 		// Note: we use a map datastore for the metadata to avoid writing / using cached migration results in the metadata store
-		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, datastore.NewMapDatastore())
+		sm, err := stmgr.NewStateManager(cs, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, datastore.NewMapDatastore(), index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}

cmd/lotus-shed/msgindex.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"path"
+
+	"github.com/ipfs/go-cid"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var msgindexCmd = &cli.Command{
+	Name:  "msgindex",
+	Usage: "Tools for managing the message index",
+	Subcommands: []*cli.Command{
+		msgindexBackfillCmd,
+		msgindexPruneCmd,
+	},
+}
+
+var msgindexBackfillCmd = &cli.Command{
+	Name:  "backfill",
+	Usage: "Backfill the message index for a number of epochs starting from a specified height",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "from",
+			Value: 0,
+			Usage: "height to start the backfill; uses the current head if omitted",
+		},
+		&cli.IntFlag{
+			Name:  "epochs",
+			Value: 1800,
+			Usage: "number of epochs to backfill; defaults to 1800 (2 finalities)",
+		},
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+			Usage: "path to the repo",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		curTs, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		startHeight := int64(cctx.Int("from"))
+		if startHeight == 0 {
+			startHeight = int64(curTs.Height()) - 1
+		}
+		epochs := cctx.Int("epochs")
+
+		basePath, err := homedir.Expand(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		dbPath := path.Join(basePath, "sqlite", "msgindex.db")
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err := db.Close()
+			if err != nil {
+				fmt.Printf("ERROR: closing db: %s", err)
+			}
+		}()
+
+		tx, err := db.Begin()
+		if err != nil {
+			return err
+		}
+
+		insertStmt, err := tx.Prepare("INSERT INTO messages VALUES (?, ?, ?)")
+		if err != nil {
+			return err
+		}
+
+		insertMsg := func(cid, tsCid cid.Cid, epoch abi.ChainEpoch) error {
+			key := cid.String()
+			tskey := tsCid.String()
+			if _, err := insertStmt.Exec(key, tskey, int64(epoch)); err != nil {
+				return err
+			}
+
+			return nil
+		}
+		rollback := func() {
+			if err := tx.Rollback(); err != nil {
+				fmt.Printf("ERROR: rollback: %s", err)
+			}
+		}
+
+		for i := 0; i < epochs; i++ {
+			epoch := abi.ChainEpoch(startHeight - int64(i))
+
+			ts, err := api.ChainGetTipSetByHeight(ctx, epoch, curTs.Key())
+			if err != nil {
+				rollback()
+				return err
+			}
+
+			tsCid, err := ts.Key().Cid()
+			if err != nil {
+				rollback()
+				return err
+			}
+
+			msgs, err := api.ChainGetMessagesInTipset(ctx, ts.Key())
+			if err != nil {
+				rollback()
+				return err
+			}
+
+			for _, msg := range msgs {
+				if err := insertMsg(msg.Cid, tsCid, epoch); err != nil {
+					rollback()
+					return err
+				}
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+var msgindexPruneCmd = &cli.Command{
+	Name:  "prune",
+	Usage: "Prune the message index for messages included before a given epoch",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "from",
+			Usage: "height to start the prune; if negative it indicates epochs from current head",
+		},
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+			Usage: "path to the repo",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		startHeight := int64(cctx.Int("from"))
+		if startHeight < 0 {
+			curTs, err := api.ChainHead(ctx)
+			if err != nil {
+				return err
+			}
+
+			startHeight += int64(curTs.Height())
+
+			if startHeight < 0 {
+				return xerrors.Errorf("bogus start height %d", startHeight)
+			}
+		}
+
+		basePath, err := homedir.Expand(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		dbPath := path.Join(basePath, "sqlite", "msgindex.db")
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err := db.Close()
+			if err != nil {
+				fmt.Printf("ERROR: closing db: %s", err)
+			}
+		}()
+
+		tx, err := db.Begin()
+		if err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec("DELETE FROM messages WHERE epoch < ?", startHeight); err != nil {
+			if err := tx.Rollback(); err != nil {
+				fmt.Printf("ERROR: rollback: %s", err)
+			}
+			return err
+		}
+
+		if err := tx.Commit(); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/rpc.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -97,7 +96,7 @@ var rpcCmd = &cli.Command{
 				return err
 			}
 
-			rb, err := ioutil.ReadAll(resp.Body)
+			rb, err := io.ReadAll(resp.Body)
 			if err != nil {
 				return err
 			}

cmd/lotus-shed/state-stats.go

@@ -26,6 +26,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -308,7 +309,7 @@ to reduce the number of decode operations performed by caching the decoded objec
 		}
 
 		tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc)
-		sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
+		sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
 		if err != nil {
 			return err
 		}

cmd/lotus-sim/simulation/node.go

@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -106,7 +107,7 @@ func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) {
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err)
 	}
-	sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil, nd.MetadataDS)
+	sim.StateManager, err = stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), us, nil, nd.MetadataDS, index.DummyMsgIndex)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err)
 	}
@@ -125,7 +126,7 @@ func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet)
 	if err != nil {
 		return nil, err
 	}
-	sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil, nd.MetadataDS)
+	sm, err := stmgr.NewStateManager(nd.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), filcns.DefaultUpgradeSchedule(), nil, nd.MetadataDS, index.DummyMsgIndex)
 	if err != nil {
 		return nil, xerrors.Errorf("creating state manager: %w", err)
 	}

cmd/lotus-sim/simulation/simulation.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
@@ -201,7 +202,7 @@ func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch
 	if err != nil {
 		return err
 	}
-	sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil, sim.Node.MetadataDS)
+	sm, err := stmgr.NewStateManager(sim.Node.Chainstore, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(mock.Verifier), newUpgradeSchedule, nil, sim.Node.MetadataDS, index.DummyMsgIndex)
 	if err != nil {
 		return err
 	}

cmd/lotus-worker/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -464,7 +463,7 @@ var runCmd = &cli.Command{
 					return xerrors.Errorf("marshaling storage config: %w", err)
 				}
 
-				if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil {
+				if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil {
 					return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err)
 				}
 

cmd/lotus-worker/storage.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -121,7 +120,7 @@ var storageAttachCmd = &cli.Command{
 				return xerrors.Errorf("marshaling storage config: %w", err)
 			}
 
-			if err := ioutil.WriteFile(filepath.Join(p, metaFile), b, 0644); err != nil {
+			if err := os.WriteFile(filepath.Join(p, metaFile), b, 0644); err != nil {
 				return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(p, metaFile), err)
 			}
 		}

cmd/lotus/daemon.go

@@ -10,9 +10,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"runtime/pprof"
 	"strings"
 
@@ -35,6 +35,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -46,6 +47,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/ulimit"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/modules/testing"
@@ -244,7 +246,7 @@ var DaemonCmd = &cli.Command{
 
 		var genBytes []byte
 		if cctx.String("genesis") != "" {
-			genBytes, err = ioutil.ReadFile(cctx.String("genesis"))
+			genBytes, err = os.ReadFile(cctx.String("genesis"))
 			if err != nil {
 				return xerrors.Errorf("reading genesis: %w", err)
 			}
@@ -400,7 +402,7 @@ func importKey(ctx context.Context, api lapi.FullNode, f string) error {
 		return err
 	}
 
-	hexdata, err := ioutil.ReadFile(f)
+	hexdata, err := os.ReadFile(f)
 	if err != nil {
 		return err
 	}
@@ -540,7 +542,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 	}
 
 	// TODO: We need to supply the actual beacon after v14
-	stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds)
+	stm, err := stmgr.NewStateManager(cst, consensus.NewTipSetExecutor(filcns.RewardFunc), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, index.DummyMsgIndex)
 	if err != nil {
 		return err
 	}
@@ -557,5 +559,23 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 		return err
 	}
 
+	// populate the message index if user has EnableMsgIndex enabled
+	//
+	c, err := lr.Config()
+	if err != nil {
+		return err
+	}
+	cfg, ok := c.(*config.FullNode)
+	if !ok {
+		return xerrors.Errorf("invalid config for repo, got: %T", c)
+	}
+	if cfg.Index.EnableMsgIndex {
+		log.Info("populating message index...")
+		if err := index.PopulateAfterSnapshot(ctx, path.Join(lr.Path(), "sqlite"), cst); err != nil {
+			return err
+		}
+		log.Info("populating message index done")
+	}
+
 	return nil
 }

conformance/corpus_test.go

@@ -5,7 +5,6 @@ package conformance
 
 import (
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -107,7 +106,7 @@ func TestConformance(t *testing.T) {
 	// Run a test for each vector.
 	for _, v := range vectors {
 		path := filepath.Join(corpusRoot, v)
-		raw, err := ioutil.ReadFile(path)
+		raw, err := os.ReadFile(path)
 		if err != nil {
 			t.Fatalf("failed to read test raw file: %s", path)
 		}

conformance/driver.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	"github.com/filecoin-project/lotus/chain/consensus/filcns"
+	"github.com/filecoin-project/lotus/chain/index"
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -108,7 +109,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, params
 
 		cs      = store.NewChainStore(bs, bs, ds, filcns.Weight, nil)
 		tse     = consensus.NewTipSetExecutor(filcns.RewardFunc)
-		sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil, ds)
+		sm, err = stmgr.NewStateManager(cs, tse, syscalls, filcns.DefaultUpgradeSchedule(), nil, ds, index.DummyMsgIndex)
 	)
 	if err != nil {
 		return nil, err

conformance/runner.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
 	"os/exec"
@@ -328,7 +327,7 @@ func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.
 // writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
 // cleaned up via t.Cleanup(). It returns the full path of the temp file.
 func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) {
-	tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
+	tmp, err := os.CreateTemp("", "lotus-tests-*.car")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err)
 	}

documentation/en/api-v1-unstable-methods.md

@@ -3941,7 +3941,9 @@ Response:
 ```
 
 ### MpoolClear
-MpoolClear clears pending messages from the mpool
+MpoolClear clears pending messages from the mpool.
+If clearLocal is true, ALL messages will be cleared.
+If clearLocal is false, local messages will be protected, all others will be cleared.
 
 
 Perms: write

documentation/en/default-lotus-config.toml

@@ -390,3 +390,11 @@
     #DatabasePath = ""
 
 
+[Index]
+  # EnableMsgIndex enables indexing of messages on chain.
+  #
+  # type: bool
+  # env var: LOTUS_INDEX_ENABLEMSGINDEX
+  #EnableMsgIndex = false
+
+

documentation/en/default-lotus-miner-config.toml

@@ -657,6 +657,16 @@
   # env var: LOTUS_SEALING_AGGREGATEABOVEBASEFEE
   #AggregateAboveBaseFee = "0.00000000032 FIL"
 
+  # When submitting several sector prove commit messages simultaneously, this option allows you to
+  # stagger the number of prove commits submitted per epoch
+  # This is done because gas estimates for ProveCommits are non deterministic and increasing as a large
+  # number of sectors get committed within the same epoch resulting in occasionally failed msgs.
+  # Submitting a smaller number of prove commits per epoch would reduce the possibility of failed msgs
+  #
+  # type: uint64
+  # env var: LOTUS_SEALING_MAXSECTORPROVECOMMITSSUBMITTEDPEREPOCH
+  #MaxSectorProveCommitsSubmittedPerEpoch = 0
+
   # type: uint64
   # env var: LOTUS_SEALING_TERMINATEBATCHMAX
   #TerminateBatchMax = 100

gateway/node.go

@@ -43,6 +43,9 @@ const (
 // TargetAPI defines the API methods that the Node depends on
 // (to make it easy to mock for tests)
 type TargetAPI interface {
+	GasEstimateGasPremium(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error)
+	StateReplay(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)
+	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error)
 	Version(context.Context) (api.APIVersion, error)
 	ChainGetParentMessages(context.Context, cid.Cid) ([]api.Message, error)
 	ChainGetParentReceipts(context.Context, cid.Cid) ([]*types.MessageReceipt, error)

gateway/proxy_fil.go

@@ -23,6 +23,36 @@ import (
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
+func (gw *Node) StateReplay(ctx context.Context, tsk types.TipSetKey, c cid.Cid) (*api.InvocResult, error) {
+	if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
+		return nil, err
+	}
+	if err := gw.checkTipsetKey(ctx, tsk); err != nil {
+		return nil, err
+	}
+	return gw.target.StateReplay(ctx, tsk, c)
+}
+
+func (gw *Node) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) {
+	if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
+		return types.BigInt{}, err
+	}
+	if err := gw.checkTipsetKey(ctx, tsk); err != nil {
+		return types.BigInt{}, err
+	}
+	return gw.target.GasEstimateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk)
+}
+
+func (gw *Node) StateMinerSectorCount(ctx context.Context, m address.Address, tsk types.TipSetKey) (api.MinerSectors, error) {
+	if err := gw.limit(ctx, chainRateLimitTokens); err != nil {
+		return api.MinerSectors{}, err
+	}
+	if err := gw.checkTipsetKey(ctx, tsk); err != nil {
+		return api.MinerSectors{}, err
+	}
+	return gw.target.StateMinerSectorCount(ctx, m, tsk)
+}
+
 func (gw *Node) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
 	return build.OpenRPCDiscoverJSON_Gateway(), nil
 }

gen/inline-gen/main.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -19,7 +18,7 @@ const (
 )
 
 func main() {
-	db, err := ioutil.ReadFile(os.Args[2])
+	db, err := os.ReadFile(os.Args[2])
 	if err != nil {
 		panic(err)
 	}
@@ -38,7 +37,7 @@ func main() {
 		if filepath.Ext(path) != ".go" {
 			return nil
 		}
-		fb, err := ioutil.ReadFile(path)
+		fb, err := os.ReadFile(path)
 		if err != nil {
 			return err
 		}
@@ -110,7 +109,7 @@ func main() {
 
 		if rewrite {
 			fmt.Printf("write %s\n", path)
-			if err := ioutil.WriteFile(path, []byte(strings.Join(outLines, "\n")), 0664); err != nil {
+			if err := os.WriteFile(path, []byte(strings.Join(outLines, "\n")), 0664); err != nil {
 				return err
 			}
 		}

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/filecoin-project/go-jsonrpc v0.2.3
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
-	github.com/filecoin-project/go-state-types v0.10.0
+	github.com/filecoin-project/go-state-types v0.11.0-alpha-3
 	github.com/filecoin-project/go-statemachine v1.0.3
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
@@ -78,7 +78,7 @@ require (
 	github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab
 	github.com/ipfs/bbloom v0.0.4
 	github.com/ipfs/go-blockservice v0.5.0
-	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/go-cid v0.4.0
 	github.com/ipfs/go-cidutil v0.1.0
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-badger2 v0.1.3
@@ -97,7 +97,7 @@ require (
 	github.com/ipfs/go-ipfs-util v0.0.2
 	github.com/ipfs/go-ipld-cbor v0.0.6
 	github.com/ipfs/go-ipld-format v0.4.0
-	github.com/ipfs/go-libipfs v0.5.0
+	github.com/ipfs/go-libipfs v0.7.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-merkledag v0.9.0
 	github.com/ipfs/go-metrics-interface v0.0.1
@@ -244,7 +244,7 @@ require (
 	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
 	github.com/ipfs/go-ipns v0.3.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-path v0.3.0 // indirect
+	github.com/ipfs/go-path v0.3.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect

go.sum

@@ -347,8 +347,8 @@ github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psS
 github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
 github.com/filecoin-project/go-state-types v0.1.8/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
 github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
-github.com/filecoin-project/go-state-types v0.10.0 h1:vsSThZIaPmOxNGG59+8D/HnlWRtlbdOjduH6ye+v8f0=
-github.com/filecoin-project/go-state-types v0.10.0/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024=
+github.com/filecoin-project/go-state-types v0.11.0-alpha-3 h1:LupJwuS2BdUEZbc0Q8N84N43O37XwTREDsdSTpzF1hg=
+github.com/filecoin-project/go-state-types v0.11.0-alpha-3/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk=
 github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54=
@@ -698,8 +698,8 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg
 github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
 github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o=
 github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro=
-github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc=
-github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw=
+github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA=
+github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
 github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q=
 github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA=
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@@ -731,7 +731,6 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
 github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
 github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
-github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo=
 github.com/ipfs/go-filestore v1.2.0 h1:O2wg7wdibwxkEDcl7xkuQsPvJFRBVgVSsOJ/GP6z3yU=
 github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHLkp9z6+FF8=
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
@@ -805,8 +804,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
 github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
 github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
-github.com/ipfs/go-libipfs v0.5.0 h1:gtvzeoTdUHPUN4B5izzyBS3Cxtpvi+l7hd2mmjN+teM=
-github.com/ipfs/go-libipfs v0.5.0/go.mod h1:iZ9QyhzNr3AkxRXrbQYb//rv7iLyvZJX0GNuc3lJDiQ=
+github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM=
+github.com/ipfs/go-libipfs v0.7.0/go.mod h1:KsIf/03CqhICzyRGyGo68tooiBE2iFbI/rXW7FhAYr0=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA=
 github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
@@ -838,17 +837,15 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
 github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks=
-github.com/ipfs/go-path v0.3.0 h1:tkjga3MtpXyM5v+3EbRvOHEoo+frwi4oumw5K+KYWyA=
-github.com/ipfs/go-path v0.3.0/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I=
+github.com/ipfs/go-path v0.3.1 h1:wkeaCWE/NTuuPGlEkLTsED5UkzfKYZpxaFFPgk8ZVLE=
+github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E=
 github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
 github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU=
 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
-github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
 github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ=
 github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM=
-github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew=
 github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=