cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #10446 from filecoin-project/steb/optimize-eth-block

Browse Source 
feat: eth: optimize eth block loading + eth_feeHistory
This commit is contained in:
raulk 2023-03-12 15:46:03 +00:00 committed by raulk
parent b332db0a38
commit d4dc28cec9
8 changed files with 123 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
chain/store/messages.go
View File

@ -237,6 +237,26 @@ func (cs *ChainStore) ReadMsgMetaCids(ctx context.Context, mmc cid.Cid) ([]cid.C
return blscids, secpkcids, nil
}
func (cs *ChainStore) ReadReceipts(ctx context.Context, root cid.Cid) ([]types.MessageReceipt, error) {
a, err := blockadt.AsArray(cs.ActorStore(ctx), root)
if err != nil {
return nil, err
}
receipts := make([]types.MessageReceipt, 0, a.Length())
var rcpt types.MessageReceipt
if err := a.ForEach(&rcpt, func(i int64) error {
if int64(len(receipts)) != i {
return xerrors.Errorf("missing receipt %d", i)
}
receipts = append(receipts, rcpt)
return nil
}); err != nil {
return nil, err
}
return receipts, nil
}
func (cs *ChainStore) MessagesForBlock(ctx context.Context, b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
blscids, secpkcids, err := cs.ReadMsgMetaCids(ctx, b.Messages)
if err != nil {

8
chain/types/ethtypes/eth_transactions.go
View File

@ -44,14 +44,6 @@ type EthTx struct {
S EthBigInt `json:"s"`
}
func (tx *EthTx) Reward(blkBaseFee big.Int) EthBigInt {
availablePriorityFee := big.Sub(big.Int(tx.MaxFeePerGas), blkBaseFee)
if big.Cmp(big.Int(tx.MaxPriorityFeePerGas), availablePriorityFee) <= 0 {
return tx.MaxPriorityFeePerGas
}
return EthBigInt(availablePriorityFee)
}
type EthTxArgs struct {
ChainID int `json:"chainId"`
Nonce int `json:"nonce"`

13
chain/types/message.go
View File

@ -215,4 +215,17 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version)
return nil
}
// EffectiveGasPremium returns the effective gas premium claimable by the miner
// given the supplied base fee.
//
// Filecoin clamps the gas premium at GasFeeCap - BaseFee, if lower than the
// specified premium.
func (m *Message) EffectiveGasPremium(baseFee abi.TokenAmount) abi.TokenAmount {
available := big.Sub(m.GasFeeCap, baseFee)
if big.Cmp(m.GasPremium, available) <= 0 {
return m.GasPremium
}
return available
}
const TestGasLimit = 100e6

3
itests/eth_block_hash_test.go
View File

@ -49,9 +49,6 @@ func TestEthBlockHashesCorrect_MultiBlockTipset(t *testing.T) {
// let the chain run a little bit longer to minimise the chance of reorgs
n2.WaitTillChain(ctx, kit.HeightAtLeast(head.Height()+50))
head, err = n2.ChainHead(context.Background())
require.NoError(t, err)
for i := 1; i <= int(head.Height()); i++ {
hex := fmt.Sprintf("0x%x", i)

8
itests/eth_conformance_test.go
View File

@ -236,14 +236,6 @@ func TestEthOpenRPCConformance(t *testing.T) {
skipReason: "earliest block is not supported",
},
{
method: "eth_getBlockByNumber",
variant: "pending",
call: func(a *ethAPIRaw) (json.RawMessage, error) {
return ethapi.EthGetBlockByNumber(context.Background(), "pending", true)
},
},
{
method: "eth_getBlockByNumber",
call: func(a *ethAPIRaw) (json.RawMessage, error) {

19
node/impl/full/chain.go
View File

@ -186,25 +186,14 @@ func (a *ChainAPI) ChainGetParentReceipts(ctx context.Context, bcid cid.Cid) ([]
return nil, nil
}
// TODO: need to get the number of messages better than this
pts, err := a.Chain.LoadTipSet(ctx, types.NewTipSetKey(b.Parents...))
receipts, err := a.Chain.ReadReceipts(ctx, b.ParentMessageReceipts)
if err != nil {
return nil, err
}
cm, err := a.Chain.MessagesForTipset(ctx, pts)
if err != nil {
return nil, err
}
var out []*types.MessageReceipt
for i := 0; i < len(cm); i++ {
r, err := a.Chain.GetParentReceipt(ctx, b, i)
if err != nil {
return nil, err
}
out = append(out, r)
out := make([]*types.MessageReceipt, len(receipts))
for i := range receipts {
out[i] = &receipts[i]
}
return out, nil

144
node/impl/full/eth.go
View File

@ -234,14 +234,13 @@ func (a *EthModule) EthGetBlockByHash(ctx context.Context, blkHash ethtypes.EthH
}
func (a *EthModule) parseBlkParam(ctx context.Context, blkParam string, strict bool) (tipset *types.TipSet, err error) {
if blkParam == "earliest" {
return nil, fmt.Errorf("block param \"earliest\" is not supported")
switch blkParam {
case "earliest", "pending":
return nil, fmt.Errorf("block param %q is not supported", blkParam)
}
head := a.Chain.GetHeaviestTipSet()
switch blkParam {
case "pending":
return head, nil
case "latest":
parent, err := a.Chain.GetTipSetFromKey(ctx, head.Parents())
if err != nil {
@ -689,49 +688,39 @@ func (a *EthModule) EthFeeHistory(ctx context.Context, p jsonrpc.RawParams) (eth
return ethtypes.EthFeeHistory{}, fmt.Errorf("bad block parameter %s: %s", params.NewestBlkNum, err)
}
oldestBlkHeight := uint64(1)
var (
basefee = ts.Blocks()[0].ParentBaseFee
oldestBlkHeight = uint64(1)
// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
// because the next base fee can be inferred from the messages in the newest block.
// However, this is NOT the case in Filecoin due to deferred execution, so the best
// we can do is duplicate the last value.
baseFeeArray := []ethtypes.EthBigInt{ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee)}
gasUsedRatioArray := []float64{}
rewardsArray := make([][]ethtypes.EthBigInt, 0)
// NOTE: baseFeePerGas should include the next block after the newest of the returned range,
// because the next base fee can be inferred from the messages in the newest block.
// However, this is NOT the case in Filecoin due to deferred execution, so the best
// we can do is duplicate the last value.
baseFeeArray = []ethtypes.EthBigInt{ethtypes.EthBigInt(basefee)}
rewardsArray = make([][]ethtypes.EthBigInt, 0)
gasUsedRatioArray = []float64{}
blocksIncluded int
)
blocksIncluded := 0
for blocksIncluded < int(params.BlkCount) && ts.Height() > 0 {
compOutput, err := a.StateCompute(ctx, ts.Height(), nil, ts.Key())
msgs, rcpts, err := messagesAndReceipts(ctx, ts, a.Chain, a.StateAPI)
if err != nil {
return ethtypes.EthFeeHistory{}, xerrors.Errorf("cannot lookup the status of tipset: %v: %w", ts, err)
return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to retrieve messages and receipts for height %d: %w", ts.Height(), err)
}
txGasRewards := gasRewardSorter{}
for _, msg := range compOutput.Trace {
if msg.Msg.From == builtintypes.SystemActorAddr {
continue
}
smsgCid, err := getSignedMessage(ctx, a.Chain, msg.MsgCid)
if err != nil {
return ethtypes.EthFeeHistory{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err)
}
tx, err := newEthTxFromSignedMessage(ctx, smsgCid, a.StateAPI)
if err != nil {
return ethtypes.EthFeeHistory{}, err
}
for i, msg := range msgs {
effectivePremium := msg.VMMessage().EffectiveGasPremium(basefee)
txGasRewards = append(txGasRewards, gasRewardTuple{
reward: tx.Reward(ts.Blocks()[0].ParentBaseFee),
gas: uint64(msg.MsgRct.GasUsed),
premium: effectivePremium,
gasUsed: rcpts[i].GasUsed,
})
}
rewards, totalGasUsed := calculateRewardsAndGasUsed(rewardPercentiles, txGasRewards)
// arrays should be reversed at the end
baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(ts.Blocks()[0].ParentBaseFee))
baseFeeArray = append(baseFeeArray, ethtypes.EthBigInt(basefee))
gasUsedRatioArray = append(gasUsedRatioArray, float64(totalGasUsed)/float64(build.BlockGasLimit))
rewardsArray = append(rewardsArray, rewards)
oldestBlkHeight = uint64(ts.Height())
@ -1792,35 +1781,33 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
return ethtypes.EthBlock{}, err
}
msgs, err := cs.MessagesForTipset(ctx, ts)
msgs, rcpts, err := messagesAndReceipts(ctx, ts, cs, sa)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
return ethtypes.EthBlock{}, xerrors.Errorf("failed to retrieve messages and receipts: %w", err)
}
block := ethtypes.NewEthBlock(len(msgs) > 0)
gasUsed := int64(0)
compOutput, err := sa.StateCompute(ctx, ts.Height(), nil, ts.Key())
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to compute state: %w", err)
}
txIdx := 0
for _, msg := range compOutput.Trace {
// skip system messages like reward application and cron
if msg.Msg.From == builtintypes.SystemActorAddr {
continue
for i, msg := range msgs {
rcpt := rcpts[i]
ti := ethtypes.EthUint64(i)
gasUsed += rcpt.GasUsed
var smsg *types.SignedMessage
switch msg := msg.(type) {
case *types.SignedMessage:
smsg = msg
case *types.Message:
smsg = &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{
Type: crypto.SigTypeBLS,
},
}
default:
return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err)
}
ti := ethtypes.EthUint64(txIdx)
txIdx++
gasUsed += msg.MsgRct.GasUsed
smsgCid, err := getSignedMessage(ctx, cs, msg.MsgCid)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.MsgCid, err)
}
tx, err := newEthTxFromSignedMessage(ctx, smsgCid, sa)
tx, err := newEthTxFromSignedMessage(ctx, smsg, sa)
if err != nil {
return ethtypes.EthBlock{}, xerrors.Errorf("failed to convert msg to ethTx: %w", err)
}
@ -1846,6 +1833,29 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx
return block, nil
}
func messagesAndReceipts(ctx context.Context, ts *types.TipSet, cs *store.ChainStore, sa StateAPI) ([]types.ChainMsg, []types.MessageReceipt, error) {
msgs, err := cs.MessagesForTipset(ctx, ts)
if err != nil {
return nil, nil, xerrors.Errorf("error loading messages for tipset: %v: %w", ts, err)
}
_, rcptRoot, err := sa.StateManager.TipSetState(ctx, ts)
if err != nil {
return nil, nil, xerrors.Errorf("failed to compute state: %w", err)
}
rcpts, err := cs.ReadReceipts(ctx, rcptRoot)
if err != nil {
return nil, nil, xerrors.Errorf("error loading receipts for tipset: %v: %w", ts, err)
}
if len(msgs) != len(rcpts) {
return nil, nil, xerrors.Errorf("receipts and message array lengths didn't match for tipset: %v: %w", ts, err)
}
return msgs, rcpts, nil
}
// lookupEthAddress makes its best effort at finding the Ethereum address for a
// Filecoin address. It does the following:
//
@ -2346,10 +2356,10 @@ func parseEthRevert(ret []byte) string {
return ethtypes.EthBytes(cbytes).String()
}
func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, uint64) {
var totalGasUsed uint64
func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRewardSorter) ([]ethtypes.EthBigInt, int64) {
var gasUsedTotal int64
for _, tx := range txGasRewards {
totalGasUsed += tx.gas
gasUsedTotal += tx.gasUsed
}
rewards := make([]ethtypes.EthBigInt, len(rewardPercentiles))
@ -2358,23 +2368,23 @@ func calculateRewardsAndGasUsed(rewardPercentiles []float64, txGasRewards gasRew
}
if len(txGasRewards) == 0 {
return rewards, totalGasUsed
return rewards, gasUsedTotal
}
sort.Stable(txGasRewards)
var idx int
var sum uint64
var sum int64
for i, percentile := range rewardPercentiles {
threshold := uint64(float64(totalGasUsed) * percentile / 100)
threshold := int64(float64(gasUsedTotal) * percentile / 100)
for sum < threshold && idx < len(txGasRewards)-1 {
sum += txGasRewards[idx].gas
sum += txGasRewards[idx].gasUsed
idx++
}
rewards[i] = txGasRewards[idx].reward
rewards[i] = ethtypes.EthBigInt(txGasRewards[idx].premium)
}
return rewards, totalGasUsed
return rewards, gasUsedTotal
}
func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid) (*types.SignedMessage, error) {
@ -2397,8 +2407,8 @@ func getSignedMessage(ctx context.Context, cs *store.ChainStore, msgCid cid.Cid)
}
type gasRewardTuple struct {
gas uint64
reward ethtypes.EthBigInt
gasUsed int64
premium abi.TokenAmount
}
// sorted in ascending order
@ -2409,5 +2419,5 @@ func (g gasRewardSorter) Swap(i, j int) {
g[i], g[j] = g[j], g[i]
}
func (g gasRewardSorter) Less(i, j int) bool {
return g[i].reward.Int.Cmp(g[j].reward.Int) == -1
return g[i].premium.Int.Cmp(g[j].premium.Int) == -1
}

21
node/impl/full/eth_test.go
View File

@ -117,11 +117,8 @@ func TestReward(t *testing.T) {
{maxFeePerGas: big.NewInt(50), maxPriorityFeePerGas: big.NewInt(200), answer: big.NewInt(-50)},
}
for _, tc := range testcases {
tx := ethtypes.EthTx{
MaxFeePerGas: ethtypes.EthBigInt(tc.maxFeePerGas),
MaxPriorityFeePerGas: ethtypes.EthBigInt(tc.maxPriorityFeePerGas),
}
reward := tx.Reward(baseFee)
msg := &types.Message{GasFeeCap: tc.maxFeePerGas, GasPremium: tc.maxPriorityFeePerGas}
reward := msg.EffectiveGasPremium(baseFee)
require.Equal(t, 0, reward.Int.Cmp(tc.answer.Int), reward, tc.answer)
}
}
@ -140,20 +137,20 @@ func TestRewardPercentiles(t *testing.T) {
{
percentiles: []float64{25, 50, 75, 100},
txGasRewards: []gasRewardTuple{
{gas: uint64(0), reward: ethtypes.EthBigInt(big.NewInt(300))},
{gas: uint64(100), reward: ethtypes.EthBigInt(big.NewInt(200))},
{gas: uint64(350), reward: ethtypes.EthBigInt(big.NewInt(100))},
{gas: uint64(500), reward: ethtypes.EthBigInt(big.NewInt(600))},
{gas: uint64(300), reward: ethtypes.EthBigInt(big.NewInt(700))},
{gasUsed: int64(0), premium: big.NewInt(300)},
{gasUsed: int64(100), premium: big.NewInt(200)},
{gasUsed: int64(350), premium: big.NewInt(100)},
{gasUsed: int64(500), premium: big.NewInt(600)},
{gasUsed: int64(300), premium: big.NewInt(700)},
},
answer: []int64{200, 700, 700, 700},
},
}
for _, tc := range testcases {
rewards, totalGasUsed := calculateRewardsAndGasUsed(tc.percentiles, tc.txGasRewards)
gasUsed := uint64(0)
var gasUsed int64
for _, tx := range tc.txGasRewards {
gasUsed += tx.gas
gasUsed += tx.gasUsed
}
ans := []ethtypes.EthBigInt{}
for _, bi := range tc.answer {