Eth JSON-RPC: populate reward in eth_feeHistory (#10245)

Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
Co-authored-by: Raúl Kripalani <raul@protocol.ai>

chain/types/ethtypes/eth_transactions.go

@@ -44,6 +44,14 @@ type EthTx struct {
 	S                    EthBigInt   `json:"s"`
 }
 
+func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt {
+	availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee)
+	if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 {
+		return tx.MaxPriorityFeePerGas
+	}
+	return EthBigInt(availablePriorityFee)
+}
+
 type EthTxArgs struct {
 	ChainID              int         `json:"chainId"`
 	Nonce                int         `json:"nonce"`

itests/eth_fee_history_test.go

@@ -37,6 +37,7 @@ func TestEthFeeHistory(t *testing.T) {
 	require.Equal(6, len(history.BaseFeePerGas))
 	require.Equal(5, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
+	require.Nil(history.Reward)
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
 		json.Marshal([]interface{}{"5", "0x10"}),
@@ -45,6 +46,7 @@ func TestEthFeeHistory(t *testing.T) {
 	require.Equal(6, len(history.BaseFeePerGas))
 	require.Equal(5, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
+	require.Nil(history.Reward)
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
 		json.Marshal([]interface{}{"0x10", "0x12"}),
@@ -53,6 +55,7 @@ func TestEthFeeHistory(t *testing.T) {
 	require.Equal(17, len(history.BaseFeePerGas))
 	require.Equal(16, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(18-16+1), history.OldestBlock)
+	require.Nil(history.Reward)
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
 		json.Marshal([]interface{}{5, "0x10"}),
@@ -61,6 +64,7 @@ func TestEthFeeHistory(t *testing.T) {
 	require.Equal(6, len(history.BaseFeePerGas))
 	require.Equal(5, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(16-5+1), history.OldestBlock)
+	require.Nil(history.Reward)
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
 		json.Marshal([]interface{}{5, "10"}),
@@ -69,19 +73,28 @@ func TestEthFeeHistory(t *testing.T) {
 	require.Equal(6, len(history.BaseFeePerGas))
 	require.Equal(5, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock)
+	require.Nil(history.Reward)
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
-		json.Marshal([]interface{}{5, "10", &[]float64{0.25, 0.50, 0.75}}),
+		json.Marshal([]interface{}{5, "10", &[]float64{25, 50, 75}}),
 	).Assert(require.NoError))
 	require.NoError(err)
 	require.Equal(6, len(history.BaseFeePerGas))
 	require.Equal(5, len(history.GasUsedRatio))
 	require.Equal(ethtypes.EthUint64(10-5+1), history.OldestBlock)
 	require.NotNil(history.Reward)
-	require.Equal(0, len(*history.Reward))
+	require.Equal(5, len(*history.Reward))
+	for _, arr := range *history.Reward {
+		require.Equal(3, len(arr))
+	}
 
 	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
-		json.Marshal([]interface{}{1025, "10", &[]float64{0.25, 0.50, 0.75}}),
+		json.Marshal([]interface{}{1025, "10", &[]float64{25, 50, 75}}),
 	).Assert(require.NoError))
 	require.Error(err)
+
+	history, err = client.EthFeeHistory(ctx, result.Wrap[jsonrpc.RawParams](
+		json.Marshal([]interface{}{5, "10", &[]float64{}}),
+	).Assert(require.NoError))
+	require.NoError(err)
 }

node/impl/full/eth.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -601,6 +602,18 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
 	if params.BlkCount > 1024 {
 		return ethtypes.EthFeeHistory{}, fmt.Errorf("block count should be smaller than 1024")
 	}
+	rewardPercentiles := make([]float64, 0)
+	if params.RewardPercentiles != nil {
+		rewardPercentiles = append(rewardPercentiles, *params.RewardPercentiles...)
+	}
+	for i, rp := range rewardPercentiles {
+		if rp < 0 || rp > 100 {
+			return ethtypes.EthFeeHistory{}, fmt.Errorf("invalid reward percentile: %f should be between 0 and 100", rp)
+		}
+		if i > 0 && rp < rewardPercentiles[i-1] {
+			return ethtypes.EthFeeHistory{}, fmt.Errorf("invalid reward percentile: %f should be larger than %f", rp, rewardPercentiles[i-1])
+		}
+	}
 
 	ts, err := a.parseBlkParam(ctx, params.NewestBlkNum)
 	if err != nil {
@@ -619,18 +632,40 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
 	//  we can do is duplicate the last value.
 	baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)}
 	gasUsedRatioArray := []float64{}
+	rewardsArray := make([][]ethtypes.EthBigInt, 0)
 
 	for ts.Height() >= abi.ChainEpoch(oldestBlkHeight) {
 		// Unfortunately we need to rebuild the full message view so we can
 		// totalize gas used in the tipset.
-		block, err := newEthBlockFromFilecoinTipSet(ctx, ts, false, a.Chain, a.StateAPI)
+		msgs, err := a.Chain.MessagesForTipset(ctx, ts)
 		if err != nil {
-			return ethtypes.EthFeeHistory{}, fmt.Errorf("cannot create eth block: %v", err)
+			return ethtypes.EthFeeHistory{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
 		}
 
-		// both arrays should be reversed at the end
+		txGasRewards := gasRewardSorter{}
+		for txIdx, msg := range msgs {
+			msgLookup, err := a.StateAPI.StateSearchMsg(ctx, types.EmptyTSK, msg.Cid(), api.LookbackNoLimit, false)
+			if err != nil || msgLookup == nil {
+				return ethtypes.EthFeeHistory{}, nil
+			}
+
+			tx, err := newEthTxFromMessageLookup(ctx, msgLookup, txIdx, a.Chain, a.StateAPI)
+			if err != nil {
+				return ethtypes.EthFeeHistory{}, nil
+			}
+
+			txGasRewards = append(txGasRewards, gasRewardTuple{
+				reward: tx.Reward(ts.Blocks()[0].ParentBaseFee),
+				gas:    uint64(msgLookup.Receipt.GasUsed),
+			})
+		}
+
+		rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards)
+
+		// arrays should be reversed at the end
 		baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee))
-		gasUsedRatioArray = append(gasUsedRatioArray, float64(block.GasUsed)/float64(build.BlockGasLimit))
+		gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit))
+		rewardsArray = append(rewardsArray, rewards)
 
 		parentTsKey := ts.Parents()
 		ts, err = a.Chain.LoadTipSet(ctx, parentTsKey)
@@ -646,6 +681,9 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
 	for i, j := 0, len(gasUsedRatioArray)-1; i < j; i, j = i+1, j-1 {
 		gasUsedRatioArray[i], gasUsedRatioArray[j] = gasUsedRatioArray[j], gasUsedRatioArray[i]
 	}
+	for i, j := 0, len(rewardsArray)-1; i < j; i, j = i+1, j-1 {
+		rewardsArray[i], rewardsArray[j] = rewardsArray[j], rewardsArray[i]
+	}
 
 	ret := ethtypes.EthFeeHistory{
 		OldestBlock:   ethtypes.EthUint64(oldestBlkHeight),
@@ -653,13 +691,7 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
 		GasUsedRatio:  gasUsedRatioArray,
 	}
 	if params.RewardPercentiles != nil {
-		// TODO: Populate reward percentiles
+		ret.Reward = &rewardsArray
-		//  https://github.com/filecoin-project/lotus/issues/10236
-		//  We need to calculate the requested percentiles of effective gas premium
-		//  based on the newest block (I presume it's the newest, we need to dig in
-		//  as it's underspecified). Effective means we're clamped at the gas_fee_cap - base_fee.
-		reward := make([][]ethtypes.EthBigInt, 0)
-		ret.Reward = &reward
 	}
 	return ret, nil
 }
@@ -2128,3 +2160,50 @@ func parseEthTopics(topics ethtypes.EthTopicSpec) (map[string][][]byte, error) {
 	}
 	return keys, nil
 }
+
+func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) {
+	var totalGasUsed uint64
+	for _, tx := range txGasRewards {
+		totalGasUsed += tx.gas
+	}
+
+	rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles))
+	for i := range rewards {
+		rewards[i] = ethtypes.EthBigIntZero
+	}
+
+	if len(txGasRewards) == 0 {
+		return rewards, totalGasUsed
+	}
+
+	sort.Stable(txGasRewards)
+
+	var idx int
+	var sum uint64
+	for i, percentile := range rewardPercentiles {
+		threshold := uint64(float64(totalGasUsed) * percentile / 100)
+		for sum < threshold && idx < len(txGasRewards)-1 {
+			sum += txGasRewards[idx].gas
+			idx++
+		}
+		rewards[i] = txGasRewards[idx].reward
+	}
+
+	return rewards, totalGasUsed
+}
+
+type gasRewardTuple struct {
+	gas    uint64
+	reward ethtypes.EthBigInt
+}
+
+// sorted in ascending order
+type gasRewardSorter []gasRewardTuple
+
+func (g gasRewardSorter) Len() int { return len(g) }
+func (g gasRewardSorter) Swap(i, j int) {
+	g[i], g[j] = g[j], g[i]
+}
+func (g gasRewardSorter) Less(i, j int) bool {
+	return g[i].reward.Int.Cmp(g[j].reward.Int) == -1
+}

node/impl/full/eth_test.go

@@ -6,6 +6,8 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/stretchr/testify/require"
 
+	"github.com/filecoin-project/go-state-types/big"
+
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 )
@@ -100,3 +102,65 @@ func TestEthLogFromEvent(t *testing.T) {
 	require.Len(t, topics, 1)
 	require.Equal(t, topics[0], ethtypes.EthHash{})
 }
+
+func TestReward(t *testing.T) {
+	baseFee := big.NewInt(100)
+	testcases := []struct {
+		maxFeePerGas, maxPriorityFeePerGas big.Int
+		answer                             big.Int
+	}{
+		{maxFeePerGas: big.NewInt(600), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(200)},
+		{maxFeePerGas: big.NewInt(600), maxPriorityFeePerGas: big.NewInt(300), answer: big.NewInt(300)},
+		{maxFeePerGas: big.NewInt(600), maxPriorityFeePerGas: big.NewInt(500), answer: big.NewInt(500)},
+		{maxFeePerGas: big.NewInt(600), maxPriorityFeePerGas: big.NewInt(600), answer: big.NewInt(500)},
+		{maxFeePerGas: big.NewInt(600), maxPriorityFeePerGas: big.NewInt(1000), answer: big.NewInt(500)},
+		{maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)},
+	}
+	for _, tc := range testcases {
+		tx := ethtypes.EthTx{
+			MaxFeePerGas:         ethtypes.EthBigInt(tc.maxFeePerGas),
+			MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas),
+		}
+		reward := tx.Reward(baseFee)
+		require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer)
+	}
+}
+
+func TestRewardPercentiles(t *testing.T) {
+	testcases := []struct {
+		percentiles  []float64
+		txGasRewards gasRewardSorter
+		answer       []int64
+	}{
+		{
+			percentiles:  []float64{25, 50, 75},
+			txGasRewards: []gasRewardTuple{},
+			answer:       []int64{0, 0, 0},
+		},
+		{
+			percentiles: []float64{25, 50, 75, 100},
+			txGasRewards: []gasRewardTuple{
+				{gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))},
+				{gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))},
+				{gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))},
+				{gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))},
+				{gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))},
+			},
+			answer: []int64{200, 700, 700, 700},
+		},
+	}
+	for _, tc := range testcases {
+		rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards)
+		gasUsed := uint64(0)
+		for _, tx := range tc.txGasRewards {
+			gasUsed += tx.gas
+		}
+		ans := []ethtypes.EthBigInt{}
+		for _, bi := range tc.answer {
+			ans = append(ans, ethtypes.EthBigInt(big.NewInt(bi)))
+		}
+		require.Equal(t, totalGasUsed, gasUsed)
+		require.Equal(t, len(ans), len(tc.percentiles))
+		require.Equal(t, ans, rewards)
+	}
+}