From ced5280571edddac81f273bc83797612461a9591 Mon Sep 17 00:00:00 2001 From: yihuang Date: Thu, 30 Dec 2021 05:47:25 +0800 Subject: [PATCH] fix: OOM when eth_getLogs response too large (#860) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: OOM when eth_getLogs response too large Closes: #858 - add limit to number of logs of filter response - make block limit and log limit configurable * return error if exceeds log limit * Apply suggestions from code review * parse from config * read cli flags * add to config template * fix bloomFilter * changelog * add validation Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com> --- CHANGELOG.md | 1 + rpc/ethereum/backend/backend.go | 74 +++---------------- rpc/ethereum/namespaces/eth/filters/api.go | 9 ++- .../namespaces/eth/filters/filters.go | 42 +++++------ rpc/ethereum/namespaces/eth/filters/utils.go | 10 ++- server/config/config.go | 20 +++++ server/config/toml.go | 5 ++ server/flags/flags.go | 20 ++--- server/start.go | 2 + 9 files changed, 82 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d356fd8..444abd04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (rpc) [tharsis#831](https://github.com/tharsis/ethermint/pull/831) Fix BaseFee value when height is specified. * (evm) [tharsis#838](https://github.com/tharsis/ethermint/pull/838) Fix splitting of trace.Memory into 32 chunks. +* (rpc) [tharsis#860](https://github.com/tharsis/ethermint/pull/860) Fix `eth_getLogs` when specify blockHash without address/topics, and limit the response size. ## [v0.9.0] - 2021-12-01 diff --git a/rpc/ethereum/backend/backend.go b/rpc/ethereum/backend/backend.go index 913990cb..1924f39a 100644 --- a/rpc/ethereum/backend/backend.go +++ b/rpc/ethereum/backend/backend.go @@ -33,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/filters" "github.com/tharsis/ethermint/rpc/ethereum/types" "github.com/tharsis/ethermint/server/config" ethermint "github.com/tharsis/ethermint/types" @@ -81,7 +80,6 @@ type Backend interface { BloomStatus() (uint64, uint64) GetLogs(hash common.Hash) ([][]*ethtypes.Log, error) GetLogsByHeight(height *int64) ([][]*ethtypes.Log, error) - GetFilteredBlocks(from int64, to int64, filter [][]filters.BloomIV, filterAddresses bool) ([]int64, error) ChainConfig() *params.ChainConfig SetTxDefaults(args evmtypes.TransactionArgs) (evmtypes.TransactionArgs, error) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx @@ -945,6 +943,16 @@ func (e *EVMBackend) RPCFeeHistoryCap() int32 { return e.cfg.JSONRPC.FeeHistoryCap } +// RPCLogsCap defines the max number of results can be returned from single `eth_getLogs` query. +func (e *EVMBackend) RPCLogsCap() int32 { + return e.cfg.JSONRPC.LogsCap +} + +// RPCBlockRangeCap defines the max block range allowed for `eth_getLogs` query. +func (e *EVMBackend) RPCBlockRangeCap() int32 { + return e.cfg.JSONRPC.BlockRangeCap +} + // RPCMinGasPrice returns the minimum gas price for a transaction obtained from // the node config. If set value is 0, it will default to 20. @@ -1017,55 +1025,6 @@ func (e *EVMBackend) BaseFee(height int64) (*big.Int, error) { return nil, nil } -// GetFilteredBlocks returns the block height list match the given bloom filters. -func (e *EVMBackend) GetFilteredBlocks( - from int64, - to int64, - filters [][]filters.BloomIV, - filterAddresses bool, -) ([]int64, error) { - matchedBlocks := make([]int64, 0) - -BLOCKS: - for height := from; height <= to; height++ { - if err := e.ctx.Err(); err != nil { - e.logger.Error("EVMBackend context error", "err", err) - return nil, err - } - - h := height - bloom, err := e.BlockBloom(&h) - if err != nil { - e.logger.Error("retrieve header failed", "blockHeight", height, "err", err) - return nil, err - } - - for i, filter := range filters { - // filter the header bloom with the addresses - if filterAddresses && i == 0 { - if !checkMatches(bloom, filter) { - continue BLOCKS - } - - // the filter doesn't have any topics - if len(filters) == 1 { - matchedBlocks = append(matchedBlocks, height) - continue BLOCKS - } - continue - } - - // filter the bloom with topics - if len(filter) > 0 && !checkMatches(bloom, filter) { - continue BLOCKS - } - } - matchedBlocks = append(matchedBlocks, height) - } - - return matchedBlocks, nil -} - // GetEthereumMsgsFromTendermintBlock returns all real MsgEthereumTxs from a Tendermint block. // It also ensures consistency over the correct txs indexes across RPC endpoints func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx { @@ -1100,16 +1059,3 @@ func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.Result return result } - -// checkMatches revised the function from -// https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L88 -func checkMatches(bloom ethtypes.Bloom, filter []filters.BloomIV) bool { - for _, bloomIV := range filter { - if bloomIV.V[0] == bloomIV.V[0]&bloom[bloomIV.I[0]] && - bloomIV.V[1] == bloomIV.V[1]&bloom[bloomIV.I[1]] && - bloomIV.V[2] == bloomIV.V[2]&bloom[bloomIV.I[2]] { - return true - } - } - return false -} diff --git a/rpc/ethereum/namespaces/eth/filters/api.go b/rpc/ethereum/namespaces/eth/filters/api.go index 2fb3a73d..9513f84b 100644 --- a/rpc/ethereum/namespaces/eth/filters/api.go +++ b/rpc/ethereum/namespaces/eth/filters/api.go @@ -29,13 +29,14 @@ type Backend interface { HeaderByHash(blockHash common.Hash) (*ethtypes.Header, error) GetLogs(blockHash common.Hash) ([][]*ethtypes.Log, error) GetLogsByNumber(blockNum types.BlockNumber) ([][]*ethtypes.Log, error) + BlockBloom(height *int64) (ethtypes.Bloom, error) GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error) BloomStatus() (uint64, uint64) - GetFilteredBlocks(from int64, to int64, bloomIndexes [][]BloomIV, filterAddresses bool) ([]int64, error) - RPCFilterCap() int32 + RPCLogsCap() int32 + RPCBlockRangeCap() int32 } // consider a filter inactive if it has not been polled for within deadline @@ -499,7 +500,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit filters.FilterCrit } // Run the filter and return all the logs - logs, err := filter.Logs(ctx) + logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap())) if err != nil { return nil, err } @@ -560,7 +561,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*et filter = NewRangeFilter(api.logger, api.backend, begin, end, f.crit.Addresses, f.crit.Topics) } // Run the filter and return all the logs - logs, err := filter.Logs(ctx) + logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap())) if err != nil { return nil, err } diff --git a/rpc/ethereum/namespaces/eth/filters/filters.go b/rpc/ethereum/namespaces/eth/filters/filters.go index 16c6f69d..e9adb5a5 100644 --- a/rpc/ethereum/namespaces/eth/filters/filters.go +++ b/rpc/ethereum/namespaces/eth/filters/filters.go @@ -84,13 +84,12 @@ func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriter } const ( - maxFilterBlocks = 100000 - maxToOverhang = 600 + maxToOverhang = 600 ) // Logs searches the blockchain for matching log entries, returning all from the // first block that contains matches, updating the start of the filter accordingly. -func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { +func (f *Filter) Logs(_ context.Context, logLimit int, blockLimit int64) ([]*ethtypes.Log, error) { logs := []*ethtypes.Log{} var err error @@ -105,7 +104,7 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { return nil, errors.Errorf("unknown block header %s", f.criteria.BlockHash.String()) } - return f.blockLogs(header) + return f.blockLogs(header.Number.Int64(), header.Bloom) } // Figure out the limits of the filter range @@ -131,8 +130,8 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { f.criteria.ToBlock = big.NewInt(1) } - if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > maxFilterBlocks { - return nil, errors.Errorf("maximum [from, to] blocks distance: %d", maxFilterBlocks) + if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > blockLimit { + return nil, errors.Errorf("maximum [from, to] blocks distance: %d", blockLimit) } // check bounds @@ -145,36 +144,37 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { from := f.criteria.FromBlock.Int64() to := f.criteria.ToBlock.Int64() - blocks, err := f.backend.GetFilteredBlocks(from, to, f.bloomFilters, len(f.criteria.Addresses) > 0) - if err != nil { - return nil, err - } - - for _, height := range blocks { - ethLogs, err := f.backend.GetLogsByNumber(types.BlockNumber(height)) + for height := from; height <= to; height++ { + bloom, err := f.backend.BlockBloom(&height) if err != nil { - return logs, errors.Wrapf(err, "failed to fetch block by number %d", height) + return nil, err } - for _, ethLog := range ethLogs { - filtered := FilterLogs(ethLog, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics) - logs = append(logs, filtered...) + filtered, err := f.blockLogs(height, bloom) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch block by number %d", height) } + + // check logs limit + if len(logs)+len(filtered) > logLimit { + return nil, errors.Errorf("query returned more than %d results", logLimit) + } + logs = append(logs, filtered...) } return logs, nil } // blockLogs returns the logs matching the filter criteria within a single block. -func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) { - if !bloomFilter(header.Bloom, f.criteria.Addresses, f.criteria.Topics) { +func (f *Filter) blockLogs(height int64, bloom ethtypes.Bloom) ([]*ethtypes.Log, error) { + if !bloomFilter(bloom, f.criteria.Addresses, f.criteria.Topics) { return []*ethtypes.Log{}, nil } // DANGER: do not call GetLogs(header.Hash()) // eth header's hash doesn't match tm block hash - logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(header.Number.Int64())) + logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(height)) if err != nil { - return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", header.Number.Int64()) + return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", height) } unfiltered := make([]*ethtypes.Log, 0) diff --git a/rpc/ethereum/namespaces/eth/filters/utils.go b/rpc/ethereum/namespaces/eth/filters/utils.go index b7362607..12a69301 100644 --- a/rpc/ethereum/namespaces/eth/filters/utils.go +++ b/rpc/ethereum/namespaces/eth/filters/utils.go @@ -57,9 +57,10 @@ func includes(addresses []common.Address, a common.Address) bool { return false } +// https://github.com/ethereum/go-ethereum/blob/v1.10.14/eth/filters/filter.go#L321 func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]common.Hash) bool { - var included bool if len(addresses) > 0 { + var included bool for _, addr := range addresses { if ethtypes.BloomLookup(bloom, addr) { included = true @@ -72,15 +73,18 @@ func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]co } for _, sub := range topics { - included = len(sub) == 0 // empty rule set == wildcard + included := len(sub) == 0 // empty rule set == wildcard for _, topic := range sub { if ethtypes.BloomLookup(bloom, topic) { included = true break } } + if !included { + return false + } } - return included + return true } // returnHashes is a helper that will return an empty hash array case the given hash array is nil, diff --git a/server/config/config.go b/server/config/config.go index 5ec1b9d2..9a018f81 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -34,6 +34,10 @@ const ( DefaultFeeHistoryCap int32 = 100 + DefaultLogsCap int32 = 10000 + + DefaultBlockRangeCap int32 = 10000 + DefaultEVMTimeout = 5 * time.Second // default 1.0 eth DefaultTxFeeCap float64 = 1.0 @@ -78,6 +82,10 @@ type JSONRPCConfig struct { FeeHistoryCap int32 `mapstructure:"feehistory-cap"` // Enable defines if the EVM RPC server should be enabled. Enable bool `mapstructure:"enable"` + // LogsCap defines the max number of results can be returned from single `eth_getLogs` query. + LogsCap int32 `mapstructure:"logs-cap"` + // BlockRangeCap defines the max block range allowed for `eth_getLogs` query. + BlockRangeCap int32 `mapstructure:"block-range-cap"` } // TLSConfig defines the certificate and matching private key for the server. @@ -171,6 +179,8 @@ func DefaultJSONRPCConfig() *JSONRPCConfig { TxFeeCap: DefaultTxFeeCap, FilterCap: DefaultFilterCap, FeeHistoryCap: DefaultFeeHistoryCap, + BlockRangeCap: DefaultBlockRangeCap, + LogsCap: DefaultLogsCap, } } @@ -196,6 +206,14 @@ func (c JSONRPCConfig) Validate() error { return errors.New("JSON-RPC EVM timeout duration cannot be negative") } + if c.LogsCap < 0 { + return errors.New("JSON-RPC logs cap cannot be negative") + } + + if c.BlockRangeCap < 0 { + return errors.New("JSON-RPC block range cap cannot be negative") + } + // TODO: validate APIs seenAPIs := make(map[string]bool) for _, api := range c.API { @@ -253,6 +271,8 @@ func GetConfig(v *viper.Viper) Config { FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"), TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"), EVMTimeout: v.GetDuration("json-rpc.evm-timeout"), + LogsCap: v.GetInt32("json-rpc.logs-cap"), + BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"), }, TLS: TLSConfig{ CertificatePath: v.GetString("tls.certificate-path"), diff --git a/server/config/toml.go b/server/config/toml.go index ff0705d1..8d108ea8 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -47,6 +47,11 @@ filter-cap = {{ .JSONRPC.FilterCap }} # FeeHistoryCap sets the global cap for total number of blocks that can be fetched feehistory-cap = {{ .JSONRPC.FeeHistoryCap }} +# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query. +logs-cap = {{ .JSONRPC.LogsCap }} + +# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query. +block-range-cap = {{ .JSONRPC.BlockRangeCap }} ############################################################################### ### TLS Configuration ### diff --git a/server/flags/flags.go b/server/flags/flags.go index f19e0d2c..9d797854 100644 --- a/server/flags/flags.go +++ b/server/flags/flags.go @@ -26,15 +26,17 @@ const ( // JSON-RPC flags const ( - JSONRPCEnable = "json-rpc.enable" - JSONRPCAPI = "json-rpc.api" - JSONRPCAddress = "json-rpc.address" - JSONWsAddress = "json-rpc.ws-address" - JSONRPCGasCap = "json-rpc.gas-cap" - JSONRPCEVMTimeout = "json-rpc.evm-timeout" - JSONRPCTxFeeCap = "json-rpc.txfee-cap" - JSONRPCFilterCap = "json-rpc.filter-cap" - JSONRPFeeHistoryCap = "json-rpc.feehistory-cap" + JSONRPCEnable = "json-rpc.enable" + JSONRPCAPI = "json-rpc.api" + JSONRPCAddress = "json-rpc.address" + JSONWsAddress = "json-rpc.ws-address" + JSONRPCGasCap = "json-rpc.gas-cap" + JSONRPCEVMTimeout = "json-rpc.evm-timeout" + JSONRPCTxFeeCap = "json-rpc.txfee-cap" + JSONRPCFilterCap = "json-rpc.filter-cap" + JSONRPFeeHistoryCap = "json-rpc.feehistory-cap" + JSONRPCLogsCap = "json-rpc.logs-cap" + JSONRPCBlockRangeCap = "json-rpc.block-range-cap" ) // EVM flags diff --git a/server/start.go b/server/start.go index 80f7b3bf..4ccd936e 100644 --- a/server/start.go +++ b/server/start.go @@ -157,6 +157,8 @@ which accepts a path for the resulting pprof file. cmd.Flags().Float64(srvflags.JSONRPCTxFeeCap, config.DefaultTxFeeCap, "Sets a cap on transaction fee that can be sent via the RPC APIs (1 = default 1 photon)") cmd.Flags().Int32(srvflags.JSONRPCFilterCap, config.DefaultFilterCap, "Sets the global cap for total number of filters that can be created") cmd.Flags().Duration(srvflags.JSONRPCEVMTimeout, config.DefaultEVMTimeout, "Sets a timeout used for eth_call (0=infinite)") + cmd.Flags().Int32(srvflags.JSONRPCLogsCap, config.DefaultLogsCap, "Sets the max number of results can be returned from single `eth_getLogs` query") + cmd.Flags().Int32(srvflags.JSONRPCBlockRangeCap, config.DefaultBlockRangeCap, "Sets the max block range allowed for `eth_getLogs` query") cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)")