Merge pull request #10446 from filecoin-project/steb/optimize-eth-block

feat: eth: optimize eth block loading + eth_feeHistory

chain/store/messages.go

@@ -237,6 +237,26 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C
 	return blscids, secpkcids, nil
 }
 
+func (cs *ChainStore) ReadReceipts(ctx context.Context, root cid.Cid) ([]types.MessageReceipt, error) {
+	a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
+	if err != nil {
+		return nil, err
+	}
+
+	receipts := make([]types.MessageReceipt, 0, a.Length())
+	var rcpt types.MessageReceipt
+	if err := a.ForEach(&rcpt, func(i int64) error {
+		if int64(len(receipts)) != i {
+			return xerrors.Errorf("missing receipt %d", i)
+		}
+		receipts = append(receipts, rcpt)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return receipts, nil
+}
+
 func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
 	blscids, secpkcids, err := cs.ReadMsgMetaCids(ctx, b.Messages)
 	if err != nil {

chain/types/ethtypes/eth_transactions.go

@@ -44,14 +44,6 @@ type EthTx struct {
 	S                    EthBigInt   `json:"s"`
 }
 
-func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt {
-	availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee)
-	if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 {
-		return tx.MaxPriorityFeePerGas
-	}
-	return EthBigInt(availablePriorityFee)
-}
-
 type EthTxArgs struct {
 	ChainID              int         `json:"chainId"`
 	Nonce                int         `json:"nonce"`

chain/types/message.go

@@ -215,4 +215,17 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version)
 	return nil
 }
 
+// EffectiveGasPremium returns the effective gas premium claimable by the miner
+// given the supplied base fee.
+//
+// Filecoin clamps the gas premium at GasFeeCap - BaseFee, if lower than the
+// specified premium.
+func (m *Message) EffectiveGasPremium(baseFee abi.TokenAmount) abi.TokenAmount {
+	available := big.Sub(m.GasFeeCap, baseFee)
+	if big.Cmp(m.GasPremium, available) <= 0 {
+		return m.GasPremium
+	}
+	return available
+}
+
 const TestGasLimit = 100e6

itests/eth_block_hash_test.go

@@ -49,9 +49,6 @@ func TestEthBlockHashesCorrect_MultiBlockTipset(t *testing.T) {
 	// let the chain run a little bit longer to minimise the chance of reorgs
 	n2.WaitTillChain(ctx, kit.HeightAtLeast(head.Height()+50))
 
-	head, err = n2.ChainHead(context.Background())
-	require.NoError(t, err)
-
 	for i := 1; i <= int(head.Height()); i++ {
 		hex := fmt.Sprintf("0x%x", i)
 

itests/eth_conformance_test.go

@@ -236,14 +236,6 @@ func TestEthOpenRPCConformance(t *testing.T) {
 			skipReason: "earliest block is not supported",
 		},
 
-		{
-			method:  "eth_getBlockByNumber",
-			variant: "pending",
-			call: func(a *ethAPIRaw) (json.RawMessage, error) {
-				return ethapi.EthGetBlockByNumber(context.Background(), "pending", true)
-			},
-		},
-
 		{
 			method: "eth_getBlockByNumber",
 			call: func(a *ethAPIRaw) (json.RawMessage, error) {

node/impl/full/chain.go

@@ -186,25 +186,14 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([]
 		return nil, nil
 	}
 
-	// TODO: need to get the number of messages better than this
+	receipts, err := a.Chain.ReadReceipts(ctx, b.ParentMessageReceipts)
-	pts, err := a.Chain.LoadTipSet(ctx, types.NewTipSetKey(b.Parents...))
 	if err != nil {
 		return nil, err
 	}
 
-	cm, err := a.Chain.MessagesForTipset(ctx, pts)
+	out := make([]*types.MessageReceipt, len(receipts))
-	if err != nil {
+	for i := range receipts {
-		return nil, err
+		out[i] = &receipts[i]
-	}
-
-	var out []*types.MessageReceipt
-	for i := 0; i < len(cm); i++ {
-		r, err := a.Chain.GetParentReceipt(ctx, b, i)
-		if err != nil {
-			return nil, err
-		}
-
-		out = append(out, r)
 	}
 
 	return out, nil

node/impl/full/eth.go

@@ -234,14 +234,13 @@ func (a *EthModule) EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthH
 }
 
 func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string, strict bool) (tipset *types.TipSet, err error) {
-	if blkParam == "earliest" {
+	switch blkParam {
-		return nil, fmt.Errorf("block param \"earliest\" is not supported")
+	case "earliest", "pending":
+		return nil, fmt.Errorf("block param %q is not supported", blkParam)
 	}
 
 	head := a.Chain.GetHeaviestTipSet()
 	switch blkParam {
-	case "pending":
-		return head, nil
 	case "latest":
 		parent, err := a.Chain.GetTipSetFromKey(ctx, head.Parents())
 		if err != nil {
@@ -689,49 +688,39 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
 		return ethtypes.EthFeeHistory{}, fmt.Errorf("bad block parameter %s: %s", params.NewestBlkNum, err)
 	}
 
-	oldestBlkHeight := uint64(1)
+	var (
+		basefee         = ts.Blocks()[0].ParentBaseFee
+		oldestBlkHeight = uint64(1)
 
-	// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
+		// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
-	//  because the next base fee can be inferred from the messages in the newest block.
+		//  because the next base fee can be inferred from the messages in the newest block.
-	//  However, this is NOT the case in Filecoin due to deferred execution, so the best
+		//  However, this is NOT the case in Filecoin due to deferred execution, so the best
-	//  we can do is duplicate the last value.
+		//  we can do is duplicate the last value.
-	baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)}
+		baseFeeArray      = []ethtypes.EthBigInt{ethtypes.EthBigInt(basefee)}
-	gasUsedRatioArray := []float64{}
+		rewardsArray      = make([][]ethtypes.EthBigInt, 0)
-	rewardsArray := make([][]ethtypes.EthBigInt, 0)
+		gasUsedRatioArray = []float64{}
+		blocksIncluded    int
+	)
 
-	blocksIncluded := 0
 	for blocksIncluded < int(params.BlkCount) && ts.Height() > 0 {
-		compOutput, err := a.StateCompute(ctx, ts.Height(), nil, ts.Key())
+		msgs, rcpts, err := messagesAndReceipts(ctx, ts, a.Chain, a.StateAPI)
 		if err != nil {
-			return ethtypes.EthFeeHistory{}, xerrors.Errorf("cannot lookup the status of tipset: %v: %w", ts, err)
+			return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to retrieve messages and receipts for height %d: %w", ts.Height(), err)
 		}
 
 		txGasRewards := gasRewardSorter{}
-		for _, msg := range compOutput.Trace {
+		for i, msg := range msgs {
-			if msg.Msg.From == builtintypes.SystemActorAddr {
+			effectivePremium := msg.VMMessage().EffectiveGasPremium(basefee)
-				continue
-			}
-
-			smsgCid, err := getSignedMessage(ctx, a.Chain, msg.MsgCid)
-			if err != nil {
-				return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err)
-			}
-
-			tx, err := newEthTxFromSignedMessage(ctx, smsgCid, a.StateAPI)
-			if err != nil {
-				return ethtypes.EthFeeHistory{}, err
-			}
-
 			txGasRewards = append(txGasRewards, gasRewardTuple{
-				reward: tx.Reward(ts.Blocks()[0].ParentBaseFee),
+				premium: effectivePremium,
-				gas:    uint64(msg.MsgRct.GasUsed),
+				gasUsed: rcpts[i].GasUsed,
 			})
 		}
 
 		rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards)
 
 		// arrays should be reversed at the end
-		baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee))
+		baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(basefee))
 		gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit))
 		rewardsArray = append(rewardsArray, rewards)
 		oldestBlkHeight = uint64(ts.Height())
@@ -1792,35 +1781,33 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
 		return ethtypes.EthBlock{}, err
 	}
 
-	msgs, err := cs.MessagesForTipset(ctx, ts)
+	msgs, rcpts, err := messagesAndReceipts(ctx, ts, cs, sa)
 	if err != nil {
-		return ethtypes.EthBlock{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
+		return ethtypes.EthBlock{}, xerrors.Errorf("failed to retrieve messages and receipts: %w", err)
 	}
 
 	block := ethtypes.NewEthBlock(len(msgs) > 0)
 
 	gasUsed := int64(0)
-	compOutput, err := sa.StateCompute(ctx, ts.Height(), nil, ts.Key())
+	for i, msg := range msgs {
-	if err != nil {
+		rcpt := rcpts[i]
-		return ethtypes.EthBlock{}, xerrors.Errorf("failed to compute state: %w", err)
+		ti := ethtypes.EthUint64(i)
-	}
+		gasUsed += rcpt.GasUsed
-
+		var smsg *types.SignedMessage
-	txIdx := 0
+		switch msg := msg.(type) {
-	for _, msg := range compOutput.Trace {
+		case *types.SignedMessage:
-		// skip system messages like reward application and cron
+			smsg = msg
-		if msg.Msg.From == builtintypes.SystemActorAddr {
+		case *types.Message:
-			continue
+			smsg = &types.SignedMessage{
+				Message: *msg,
+				Signature: crypto.Signature{
+					Type: crypto.SigTypeBLS,
+				},
+			}
+		default:
+			return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err)
 		}
-
+		tx, err := newEthTxFromSignedMessage(ctx, smsg, sa)
-		ti := ethtypes.EthUint64(txIdx)
-		txIdx++
-
-		gasUsed += msg.MsgRct.GasUsed
-		smsgCid, err := getSignedMessage(ctx, cs, msg.MsgCid)
-		if err != nil {
-			return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err)
-		}
-		tx, err := newEthTxFromSignedMessage(ctx, smsgCid, sa)
 		if err != nil {
 			return ethtypes.EthBlock{}, xerrors.Errorf("failed to convert msg to ethTx: %w", err)
 		}
@@ -1846,6 +1833,29 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
 	return block, nil
 }
 
+func messagesAndReceipts(ctx context.Context, ts *types.TipSet, cs *store.ChainStore, sa StateAPI) ([]types.ChainMsg, []types.MessageReceipt, error) {
+	msgs, err := cs.MessagesForTipset(ctx, ts)
+	if err != nil {
+		return nil, nil, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
+	}
+
+	_, rcptRoot, err := sa.StateManager.TipSetState(ctx, ts)
+	if err != nil {
+		return nil, nil, xerrors.Errorf("failed to compute state: %w", err)
+	}
+
+	rcpts, err := cs.ReadReceipts(ctx, rcptRoot)
+	if err != nil {
+		return nil, nil, xerrors.Errorf("error loading receipts for tipset: %v: %w", ts, err)
+	}
+
+	if len(msgs) != len(rcpts) {
+		return nil, nil, xerrors.Errorf("receipts and message array lengths didn't match for tipset: %v: %w", ts, err)
+	}
+
+	return msgs, rcpts, nil
+}
+
 // lookupEthAddress makes its best effort at finding the Ethereum address for a
 // Filecoin address. It does the following:
 //
@@ -2346,10 +2356,10 @@ func parseEthRevert(ret []byte) string {
 	return ethtypes.EthBytes(cbytes).String()
 }
 
-func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) {
+func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, int64) {
-	var totalGasUsed uint64
+	var gasUsedTotal int64
 	for _, tx := range txGasRewards {
-		totalGasUsed += tx.gas
+		gasUsedTotal += tx.gasUsed
 	}
 
 	rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles))
@@ -2358,23 +2368,23 @@ func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRew
 	}
 
 	if len(txGasRewards) == 0 {
-		return rewards, totalGasUsed
+		return rewards, gasUsedTotal
 	}
 
 	sort.Stable(txGasRewards)
 
 	var idx int
-	var sum uint64
+	var sum int64
 	for i, percentile := range rewardPercentiles {
-		threshold := uint64(float64(totalGasUsed) * percentile / 100)
+		threshold := int64(float64(gasUsedTotal) * percentile / 100)
 		for sum < threshold && idx < len(txGasRewards)-1 {
-			sum += txGasRewards[idx].gas
+			sum += txGasRewards[idx].gasUsed
 			idx++
 		}
-		rewards[i] = txGasRewards[idx].reward
+		rewards[i] = ethtypes.EthBigInt(txGasRewards[idx].premium)
 	}
 
-	return rewards, totalGasUsed
+	return rewards, gasUsedTotal
 }
 
 func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) {
@@ -2397,8 +2407,8 @@ func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid)
 }
 
 type gasRewardTuple struct {
-	gas    uint64
+	gasUsed int64
-	reward ethtypes.EthBigInt
+	premium abi.TokenAmount
 }
 
 // sorted in ascending order
@@ -2409,5 +2419,5 @@ func (g gasRewardSorter) Swap(i, j int) {
 	g[i], g[j] = g[j], g[i]
 }
 func (g gasRewardSorter) Less(i, j int) bool {
-	return g[i].reward.Int.Cmp(g[j].reward.Int) == -1
+	return g[i].premium.Int.Cmp(g[j].premium.Int) == -1
 }

node/impl/full/eth_test.go

@@ -117,11 +117,8 @@ func TestReward(t *testing.T) {
 		{maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)},
 	}
 	for _, tc := range testcases {
-		tx := ethtypes.EthTx{
+		msg := &types.Message{GasFeeCap: tc.maxFeePerGas, GasPremium: tc.maxPriorityFeePerGas}
-			MaxFeePerGas:         ethtypes.EthBigInt(tc.maxFeePerGas),
+		reward := msg.EffectiveGasPremium(baseFee)
-			MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas),
-		}
-		reward := tx.Reward(baseFee)
 		require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer)
 	}
 }
@@ -140,20 +137,20 @@ func TestRewardPercentiles(t *testing.T) {
 		{
 			percentiles: []float64{25, 50, 75, 100},
 			txGasRewards: []gasRewardTuple{
-				{gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))},
+				{gasUsed: int64(0), premium: big.NewInt(300)},
-				{gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))},
+				{gasUsed: int64(100), premium: big.NewInt(200)},
-				{gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))},
+				{gasUsed: int64(350), premium: big.NewInt(100)},
-				{gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))},
+				{gasUsed: int64(500), premium: big.NewInt(600)},
-				{gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))},
+				{gasUsed: int64(300), premium: big.NewInt(700)},
 			},
 			answer: []int64{200, 700, 700, 700},
 		},
 	}
 	for _, tc := range testcases {
 		rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards)
-		gasUsed := uint64(0)
+		var gasUsed int64
 		for _, tx := range tc.txGasRewards {
-			gasUsed += tx.gas
+			gasUsed += tx.gasUsed
 		}
 		ans := []ethtypes.EthBigInt{}
 		for _, bi := range tc.answer {