fix: OOM when eth_getLogs response too large (#860)

* fix: OOM when eth_getLogs response too large

Closes: #858

- add limit to number of logs of filter response
- make block limit and log limit configurable

* return error if exceeds log limit

* Apply suggestions from code review

* parse from config

* read cli flags

* add to config template

* fix bloomFilter

* changelog

* add validation

Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
This commit is contained in:
yihuang 2021-12-30 05:47:25 +08:00 committed by GitHub
parent fdfe3d9761
commit ced5280571
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 82 additions and 101 deletions

View File

@ -58,6 +58,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* (rpc) [tharsis#831](https://github.com/tharsis/ethermint/pull/831) Fix BaseFee value when height is specified. * (rpc) [tharsis#831](https://github.com/tharsis/ethermint/pull/831) Fix BaseFee value when height is specified.
* (evm) [tharsis#838](https://github.com/tharsis/ethermint/pull/838) Fix splitting of trace.Memory into 32 chunks. * (evm) [tharsis#838](https://github.com/tharsis/ethermint/pull/838) Fix splitting of trace.Memory into 32 chunks.
* (rpc) [tharsis#860](https://github.com/tharsis/ethermint/pull/860) Fix `eth_getLogs` when specify blockHash without address/topics, and limit the response size.
## [v0.9.0] - 2021-12-01 ## [v0.9.0] - 2021-12-01

View File

@ -33,7 +33,6 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/hexutil"
ethtypes "github.com/ethereum/go-ethereum/core/types" ethtypes "github.com/ethereum/go-ethereum/core/types"
"github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/filters"
"github.com/tharsis/ethermint/rpc/ethereum/types" "github.com/tharsis/ethermint/rpc/ethereum/types"
"github.com/tharsis/ethermint/server/config" "github.com/tharsis/ethermint/server/config"
ethermint "github.com/tharsis/ethermint/types" ethermint "github.com/tharsis/ethermint/types"
@ -81,7 +80,6 @@ type Backend interface {
BloomStatus() (uint64, uint64) BloomStatus() (uint64, uint64)
GetLogs(hash common.Hash) ([][]*ethtypes.Log, error) GetLogs(hash common.Hash) ([][]*ethtypes.Log, error)
GetLogsByHeight(height *int64) ([][]*ethtypes.Log, error) GetLogsByHeight(height *int64) ([][]*ethtypes.Log, error)
GetFilteredBlocks(from int64, to int64, filter [][]filters.BloomIV, filterAddresses bool) ([]int64, error)
ChainConfig() *params.ChainConfig ChainConfig() *params.ChainConfig
SetTxDefaults(args evmtypes.TransactionArgs) (evmtypes.TransactionArgs, error) SetTxDefaults(args evmtypes.TransactionArgs) (evmtypes.TransactionArgs, error)
GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx
@ -945,6 +943,16 @@ func (e *EVMBackend) RPCFeeHistoryCap() int32 {
return e.cfg.JSONRPC.FeeHistoryCap return e.cfg.JSONRPC.FeeHistoryCap
} }
// RPCLogsCap defines the max number of results can be returned from single `eth_getLogs` query.
func (e *EVMBackend) RPCLogsCap() int32 {
return e.cfg.JSONRPC.LogsCap
}
// RPCBlockRangeCap defines the max block range allowed for `eth_getLogs` query.
func (e *EVMBackend) RPCBlockRangeCap() int32 {
return e.cfg.JSONRPC.BlockRangeCap
}
// RPCMinGasPrice returns the minimum gas price for a transaction obtained from // RPCMinGasPrice returns the minimum gas price for a transaction obtained from
// the node config. If set value is 0, it will default to 20. // the node config. If set value is 0, it will default to 20.
@ -1017,55 +1025,6 @@ func (e *EVMBackend) BaseFee(height int64) (*big.Int, error) {
return nil, nil return nil, nil
} }
// GetFilteredBlocks returns the block height list match the given bloom filters.
func (e *EVMBackend) GetFilteredBlocks(
from int64,
to int64,
filters [][]filters.BloomIV,
filterAddresses bool,
) ([]int64, error) {
matchedBlocks := make([]int64, 0)
BLOCKS:
for height := from; height <= to; height++ {
if err := e.ctx.Err(); err != nil {
e.logger.Error("EVMBackend context error", "err", err)
return nil, err
}
h := height
bloom, err := e.BlockBloom(&h)
if err != nil {
e.logger.Error("retrieve header failed", "blockHeight", height, "err", err)
return nil, err
}
for i, filter := range filters {
// filter the header bloom with the addresses
if filterAddresses && i == 0 {
if !checkMatches(bloom, filter) {
continue BLOCKS
}
// the filter doesn't have any topics
if len(filters) == 1 {
matchedBlocks = append(matchedBlocks, height)
continue BLOCKS
}
continue
}
// filter the bloom with topics
if len(filter) > 0 && !checkMatches(bloom, filter) {
continue BLOCKS
}
}
matchedBlocks = append(matchedBlocks, height)
}
return matchedBlocks, nil
}
// GetEthereumMsgsFromTendermintBlock returns all real MsgEthereumTxs from a Tendermint block. // GetEthereumMsgsFromTendermintBlock returns all real MsgEthereumTxs from a Tendermint block.
// It also ensures consistency over the correct txs indexes across RPC endpoints // It also ensures consistency over the correct txs indexes across RPC endpoints
func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx { func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx {
@ -1100,16 +1059,3 @@ func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.Result
return result return result
} }
// checkMatches revised the function from
// https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L88
func checkMatches(bloom ethtypes.Bloom, filter []filters.BloomIV) bool {
for _, bloomIV := range filter {
if bloomIV.V[0] == bloomIV.V[0]&bloom[bloomIV.I[0]] &&
bloomIV.V[1] == bloomIV.V[1]&bloom[bloomIV.I[1]] &&
bloomIV.V[2] == bloomIV.V[2]&bloom[bloomIV.I[2]] {
return true
}
}
return false
}

View File

@ -29,13 +29,14 @@ type Backend interface {
HeaderByHash(blockHash common.Hash) (*ethtypes.Header, error) HeaderByHash(blockHash common.Hash) (*ethtypes.Header, error)
GetLogs(blockHash common.Hash) ([][]*ethtypes.Log, error) GetLogs(blockHash common.Hash) ([][]*ethtypes.Log, error)
GetLogsByNumber(blockNum types.BlockNumber) ([][]*ethtypes.Log, error) GetLogsByNumber(blockNum types.BlockNumber) ([][]*ethtypes.Log, error)
BlockBloom(height *int64) (ethtypes.Bloom, error)
GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error) GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error)
BloomStatus() (uint64, uint64) BloomStatus() (uint64, uint64)
GetFilteredBlocks(from int64, to int64, bloomIndexes [][]BloomIV, filterAddresses bool) ([]int64, error)
RPCFilterCap() int32 RPCFilterCap() int32
RPCLogsCap() int32
RPCBlockRangeCap() int32
} }
// consider a filter inactive if it has not been polled for within deadline // consider a filter inactive if it has not been polled for within deadline
@ -499,7 +500,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit filters.FilterCrit
} }
// Run the filter and return all the logs // Run the filter and return all the logs
logs, err := filter.Logs(ctx) logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -560,7 +561,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*et
filter = NewRangeFilter(api.logger, api.backend, begin, end, f.crit.Addresses, f.crit.Topics) filter = NewRangeFilter(api.logger, api.backend, begin, end, f.crit.Addresses, f.crit.Topics)
} }
// Run the filter and return all the logs // Run the filter and return all the logs
logs, err := filter.Logs(ctx) logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -84,13 +84,12 @@ func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriter
} }
const ( const (
maxFilterBlocks = 100000
maxToOverhang = 600 maxToOverhang = 600
) )
// Logs searches the blockchain for matching log entries, returning all from the // Logs searches the blockchain for matching log entries, returning all from the
// first block that contains matches, updating the start of the filter accordingly. // first block that contains matches, updating the start of the filter accordingly.
func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { func (f *Filter) Logs(_ context.Context, logLimit int, blockLimit int64) ([]*ethtypes.Log, error) {
logs := []*ethtypes.Log{} logs := []*ethtypes.Log{}
var err error var err error
@ -105,7 +104,7 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
return nil, errors.Errorf("unknown block header %s", f.criteria.BlockHash.String()) return nil, errors.Errorf("unknown block header %s", f.criteria.BlockHash.String())
} }
return f.blockLogs(header) return f.blockLogs(header.Number.Int64(), header.Bloom)
} }
// Figure out the limits of the filter range // Figure out the limits of the filter range
@ -131,8 +130,8 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
f.criteria.ToBlock = big.NewInt(1) f.criteria.ToBlock = big.NewInt(1)
} }
if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > maxFilterBlocks { if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > blockLimit {
return nil, errors.Errorf("maximum [from, to] blocks distance: %d", maxFilterBlocks) return nil, errors.Errorf("maximum [from, to] blocks distance: %d", blockLimit)
} }
// check bounds // check bounds
@ -145,36 +144,37 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
from := f.criteria.FromBlock.Int64() from := f.criteria.FromBlock.Int64()
to := f.criteria.ToBlock.Int64() to := f.criteria.ToBlock.Int64()
blocks, err := f.backend.GetFilteredBlocks(from, to, f.bloomFilters, len(f.criteria.Addresses) > 0) for height := from; height <= to; height++ {
bloom, err := f.backend.BlockBloom(&height)
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, height := range blocks { filtered, err := f.blockLogs(height, bloom)
ethLogs, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
if err != nil { if err != nil {
return logs, errors.Wrapf(err, "failed to fetch block by number %d", height) return nil, errors.Wrapf(err, "failed to fetch block by number %d", height)
} }
for _, ethLog := range ethLogs { // check logs limit
filtered := FilterLogs(ethLog, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics) if len(logs)+len(filtered) > logLimit {
logs = append(logs, filtered...) return nil, errors.Errorf("query returned more than %d results", logLimit)
} }
logs = append(logs, filtered...)
} }
return logs, nil return logs, nil
} }
// blockLogs returns the logs matching the filter criteria within a single block. // blockLogs returns the logs matching the filter criteria within a single block.
func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) { func (f *Filter) blockLogs(height int64, bloom ethtypes.Bloom) ([]*ethtypes.Log, error) {
if !bloomFilter(header.Bloom, f.criteria.Addresses, f.criteria.Topics) { if !bloomFilter(bloom, f.criteria.Addresses, f.criteria.Topics) {
return []*ethtypes.Log{}, nil return []*ethtypes.Log{}, nil
} }
// DANGER: do not call GetLogs(header.Hash()) // DANGER: do not call GetLogs(header.Hash())
// eth header's hash doesn't match tm block hash // eth header's hash doesn't match tm block hash
logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(header.Number.Int64())) logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
if err != nil { if err != nil {
return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", header.Number.Int64()) return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", height)
} }
unfiltered := make([]*ethtypes.Log, 0) unfiltered := make([]*ethtypes.Log, 0)

View File

@ -57,9 +57,10 @@ func includes(addresses []common.Address, a common.Address) bool {
return false return false
} }
// https://github.com/ethereum/go-ethereum/blob/v1.10.14/eth/filters/filter.go#L321
func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]common.Hash) bool { func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
var included bool
if len(addresses) > 0 { if len(addresses) > 0 {
var included bool
for _, addr := range addresses { for _, addr := range addresses {
if ethtypes.BloomLookup(bloom, addr) { if ethtypes.BloomLookup(bloom, addr) {
included = true included = true
@ -72,15 +73,18 @@ func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]co
} }
for _, sub := range topics { for _, sub := range topics {
included = len(sub) == 0 // empty rule set == wildcard included := len(sub) == 0 // empty rule set == wildcard
for _, topic := range sub { for _, topic := range sub {
if ethtypes.BloomLookup(bloom, topic) { if ethtypes.BloomLookup(bloom, topic) {
included = true included = true
break break
} }
} }
if !included {
return false
} }
return included }
return true
} }
// returnHashes is a helper that will return an empty hash array case the given hash array is nil, // returnHashes is a helper that will return an empty hash array case the given hash array is nil,

View File

@ -34,6 +34,10 @@ const (
DefaultFeeHistoryCap int32 = 100 DefaultFeeHistoryCap int32 = 100
DefaultLogsCap int32 = 10000
DefaultBlockRangeCap int32 = 10000
DefaultEVMTimeout = 5 * time.Second DefaultEVMTimeout = 5 * time.Second
// default 1.0 eth // default 1.0 eth
DefaultTxFeeCap float64 = 1.0 DefaultTxFeeCap float64 = 1.0
@ -78,6 +82,10 @@ type JSONRPCConfig struct {
FeeHistoryCap int32 `mapstructure:"feehistory-cap"` FeeHistoryCap int32 `mapstructure:"feehistory-cap"`
// Enable defines if the EVM RPC server should be enabled. // Enable defines if the EVM RPC server should be enabled.
Enable bool `mapstructure:"enable"` Enable bool `mapstructure:"enable"`
// LogsCap defines the max number of results can be returned from single `eth_getLogs` query.
LogsCap int32 `mapstructure:"logs-cap"`
// BlockRangeCap defines the max block range allowed for `eth_getLogs` query.
BlockRangeCap int32 `mapstructure:"block-range-cap"`
} }
// TLSConfig defines the certificate and matching private key for the server. // TLSConfig defines the certificate and matching private key for the server.
@ -171,6 +179,8 @@ func DefaultJSONRPCConfig() *JSONRPCConfig {
TxFeeCap: DefaultTxFeeCap, TxFeeCap: DefaultTxFeeCap,
FilterCap: DefaultFilterCap, FilterCap: DefaultFilterCap,
FeeHistoryCap: DefaultFeeHistoryCap, FeeHistoryCap: DefaultFeeHistoryCap,
BlockRangeCap: DefaultBlockRangeCap,
LogsCap: DefaultLogsCap,
} }
} }
@ -196,6 +206,14 @@ func (c JSONRPCConfig) Validate() error {
return errors.New("JSON-RPC EVM timeout duration cannot be negative") return errors.New("JSON-RPC EVM timeout duration cannot be negative")
} }
if c.LogsCap < 0 {
return errors.New("JSON-RPC logs cap cannot be negative")
}
if c.BlockRangeCap < 0 {
return errors.New("JSON-RPC block range cap cannot be negative")
}
// TODO: validate APIs // TODO: validate APIs
seenAPIs := make(map[string]bool) seenAPIs := make(map[string]bool)
for _, api := range c.API { for _, api := range c.API {
@ -253,6 +271,8 @@ func GetConfig(v *viper.Viper) Config {
FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"), FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"),
TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"), TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"),
EVMTimeout: v.GetDuration("json-rpc.evm-timeout"), EVMTimeout: v.GetDuration("json-rpc.evm-timeout"),
LogsCap: v.GetInt32("json-rpc.logs-cap"),
BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"),
}, },
TLS: TLSConfig{ TLS: TLSConfig{
CertificatePath: v.GetString("tls.certificate-path"), CertificatePath: v.GetString("tls.certificate-path"),

View File

@ -47,6 +47,11 @@ filter-cap = {{ .JSONRPC.FilterCap }}
# FeeHistoryCap sets the global cap for total number of blocks that can be fetched # FeeHistoryCap sets the global cap for total number of blocks that can be fetched
feehistory-cap = {{ .JSONRPC.FeeHistoryCap }} feehistory-cap = {{ .JSONRPC.FeeHistoryCap }}
# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
logs-cap = {{ .JSONRPC.LogsCap }}
# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
block-range-cap = {{ .JSONRPC.BlockRangeCap }}
############################################################################### ###############################################################################
### TLS Configuration ### ### TLS Configuration ###

View File

@ -35,6 +35,8 @@ const (
JSONRPCTxFeeCap = "json-rpc.txfee-cap" JSONRPCTxFeeCap = "json-rpc.txfee-cap"
JSONRPCFilterCap = "json-rpc.filter-cap" JSONRPCFilterCap = "json-rpc.filter-cap"
JSONRPFeeHistoryCap = "json-rpc.feehistory-cap" JSONRPFeeHistoryCap = "json-rpc.feehistory-cap"
JSONRPCLogsCap = "json-rpc.logs-cap"
JSONRPCBlockRangeCap = "json-rpc.block-range-cap"
) )
// EVM flags // EVM flags

View File

@ -157,6 +157,8 @@ which accepts a path for the resulting pprof file.
cmd.Flags().Float64(srvflags.JSONRPCTxFeeCap, config.DefaultTxFeeCap, "Sets a cap on transaction fee that can be sent via the RPC APIs (1 = default 1 photon)") cmd.Flags().Float64(srvflags.JSONRPCTxFeeCap, config.DefaultTxFeeCap, "Sets a cap on transaction fee that can be sent via the RPC APIs (1 = default 1 photon)")
cmd.Flags().Int32(srvflags.JSONRPCFilterCap, config.DefaultFilterCap, "Sets the global cap for total number of filters that can be created") cmd.Flags().Int32(srvflags.JSONRPCFilterCap, config.DefaultFilterCap, "Sets the global cap for total number of filters that can be created")
cmd.Flags().Duration(srvflags.JSONRPCEVMTimeout, config.DefaultEVMTimeout, "Sets a timeout used for eth_call (0=infinite)") cmd.Flags().Duration(srvflags.JSONRPCEVMTimeout, config.DefaultEVMTimeout, "Sets a timeout used for eth_call (0=infinite)")
cmd.Flags().Int32(srvflags.JSONRPCLogsCap, config.DefaultLogsCap, "Sets the max number of results can be returned from single `eth_getLogs` query")
cmd.Flags().Int32(srvflags.JSONRPCBlockRangeCap, config.DefaultBlockRangeCap, "Sets the max block range allowed for `eth_getLogs` query")
cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)") cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)")