forked from cerc-io/laconicd-deprecated
fix: OOM when eth_getLogs response too large (#860)
* fix: OOM when eth_getLogs response too large Closes: #858 - add limit to number of logs of filter response - make block limit and log limit configurable * return error if exceeds log limit * Apply suggestions from code review * parse from config * read cli flags * add to config template * fix bloomFilter * changelog * add validation Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
This commit is contained in:
parent
fdfe3d9761
commit
ced5280571
@ -58,6 +58,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
|
||||
|
||||
* (rpc) [tharsis#831](https://github.com/tharsis/ethermint/pull/831) Fix BaseFee value when height is specified.
|
||||
* (evm) [tharsis#838](https://github.com/tharsis/ethermint/pull/838) Fix splitting of trace.Memory into 32 chunks.
|
||||
* (rpc) [tharsis#860](https://github.com/tharsis/ethermint/pull/860) Fix `eth_getLogs` when specify blockHash without address/topics, and limit the response size.
|
||||
|
||||
## [v0.9.0] - 2021-12-01
|
||||
|
||||
|
@ -33,7 +33,6 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
ethtypes "github.com/ethereum/go-ethereum/core/types"
|
||||
|
||||
"github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/filters"
|
||||
"github.com/tharsis/ethermint/rpc/ethereum/types"
|
||||
"github.com/tharsis/ethermint/server/config"
|
||||
ethermint "github.com/tharsis/ethermint/types"
|
||||
@ -81,7 +80,6 @@ type Backend interface {
|
||||
BloomStatus() (uint64, uint64)
|
||||
GetLogs(hash common.Hash) ([][]*ethtypes.Log, error)
|
||||
GetLogsByHeight(height *int64) ([][]*ethtypes.Log, error)
|
||||
GetFilteredBlocks(from int64, to int64, filter [][]filters.BloomIV, filterAddresses bool) ([]int64, error)
|
||||
ChainConfig() *params.ChainConfig
|
||||
SetTxDefaults(args evmtypes.TransactionArgs) (evmtypes.TransactionArgs, error)
|
||||
GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx
|
||||
@ -945,6 +943,16 @@ func (e *EVMBackend) RPCFeeHistoryCap() int32 {
|
||||
return e.cfg.JSONRPC.FeeHistoryCap
|
||||
}
|
||||
|
||||
// RPCLogsCap defines the max number of results can be returned from single `eth_getLogs` query.
|
||||
func (e *EVMBackend) RPCLogsCap() int32 {
|
||||
return e.cfg.JSONRPC.LogsCap
|
||||
}
|
||||
|
||||
// RPCBlockRangeCap defines the max block range allowed for `eth_getLogs` query.
|
||||
func (e *EVMBackend) RPCBlockRangeCap() int32 {
|
||||
return e.cfg.JSONRPC.BlockRangeCap
|
||||
}
|
||||
|
||||
// RPCMinGasPrice returns the minimum gas price for a transaction obtained from
|
||||
// the node config. If set value is 0, it will default to 20.
|
||||
|
||||
@ -1017,55 +1025,6 @@ func (e *EVMBackend) BaseFee(height int64) (*big.Int, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// GetFilteredBlocks returns the block height list match the given bloom filters.
|
||||
func (e *EVMBackend) GetFilteredBlocks(
|
||||
from int64,
|
||||
to int64,
|
||||
filters [][]filters.BloomIV,
|
||||
filterAddresses bool,
|
||||
) ([]int64, error) {
|
||||
matchedBlocks := make([]int64, 0)
|
||||
|
||||
BLOCKS:
|
||||
for height := from; height <= to; height++ {
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
e.logger.Error("EVMBackend context error", "err", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h := height
|
||||
bloom, err := e.BlockBloom(&h)
|
||||
if err != nil {
|
||||
e.logger.Error("retrieve header failed", "blockHeight", height, "err", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, filter := range filters {
|
||||
// filter the header bloom with the addresses
|
||||
if filterAddresses && i == 0 {
|
||||
if !checkMatches(bloom, filter) {
|
||||
continue BLOCKS
|
||||
}
|
||||
|
||||
// the filter doesn't have any topics
|
||||
if len(filters) == 1 {
|
||||
matchedBlocks = append(matchedBlocks, height)
|
||||
continue BLOCKS
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// filter the bloom with topics
|
||||
if len(filter) > 0 && !checkMatches(bloom, filter) {
|
||||
continue BLOCKS
|
||||
}
|
||||
}
|
||||
matchedBlocks = append(matchedBlocks, height)
|
||||
}
|
||||
|
||||
return matchedBlocks, nil
|
||||
}
|
||||
|
||||
// GetEthereumMsgsFromTendermintBlock returns all real MsgEthereumTxs from a Tendermint block.
|
||||
// It also ensures consistency over the correct txs indexes across RPC endpoints
|
||||
func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock, blockRes *tmrpctypes.ResultBlockResults) []*evmtypes.MsgEthereumTx {
|
||||
@ -1100,16 +1059,3 @@ func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.Result
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// checkMatches revised the function from
|
||||
// https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L88
|
||||
func checkMatches(bloom ethtypes.Bloom, filter []filters.BloomIV) bool {
|
||||
for _, bloomIV := range filter {
|
||||
if bloomIV.V[0] == bloomIV.V[0]&bloom[bloomIV.I[0]] &&
|
||||
bloomIV.V[1] == bloomIV.V[1]&bloom[bloomIV.I[1]] &&
|
||||
bloomIV.V[2] == bloomIV.V[2]&bloom[bloomIV.I[2]] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
@ -29,13 +29,14 @@ type Backend interface {
|
||||
HeaderByHash(blockHash common.Hash) (*ethtypes.Header, error)
|
||||
GetLogs(blockHash common.Hash) ([][]*ethtypes.Log, error)
|
||||
GetLogsByNumber(blockNum types.BlockNumber) ([][]*ethtypes.Log, error)
|
||||
BlockBloom(height *int64) (ethtypes.Bloom, error)
|
||||
|
||||
GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error)
|
||||
BloomStatus() (uint64, uint64)
|
||||
|
||||
GetFilteredBlocks(from int64, to int64, bloomIndexes [][]BloomIV, filterAddresses bool) ([]int64, error)
|
||||
|
||||
RPCFilterCap() int32
|
||||
RPCLogsCap() int32
|
||||
RPCBlockRangeCap() int32
|
||||
}
|
||||
|
||||
// consider a filter inactive if it has not been polled for within deadline
|
||||
@ -499,7 +500,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit filters.FilterCrit
|
||||
}
|
||||
|
||||
// Run the filter and return all the logs
|
||||
logs, err := filter.Logs(ctx)
|
||||
logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -560,7 +561,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*et
|
||||
filter = NewRangeFilter(api.logger, api.backend, begin, end, f.crit.Addresses, f.crit.Topics)
|
||||
}
|
||||
// Run the filter and return all the logs
|
||||
logs, err := filter.Logs(ctx)
|
||||
logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -84,13 +84,12 @@ func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriter
|
||||
}
|
||||
|
||||
const (
|
||||
maxFilterBlocks = 100000
|
||||
maxToOverhang = 600
|
||||
maxToOverhang = 600
|
||||
)
|
||||
|
||||
// Logs searches the blockchain for matching log entries, returning all from the
|
||||
// first block that contains matches, updating the start of the filter accordingly.
|
||||
func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
|
||||
func (f *Filter) Logs(_ context.Context, logLimit int, blockLimit int64) ([]*ethtypes.Log, error) {
|
||||
logs := []*ethtypes.Log{}
|
||||
var err error
|
||||
|
||||
@ -105,7 +104,7 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
|
||||
return nil, errors.Errorf("unknown block header %s", f.criteria.BlockHash.String())
|
||||
}
|
||||
|
||||
return f.blockLogs(header)
|
||||
return f.blockLogs(header.Number.Int64(), header.Bloom)
|
||||
}
|
||||
|
||||
// Figure out the limits of the filter range
|
||||
@ -131,8 +130,8 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
|
||||
f.criteria.ToBlock = big.NewInt(1)
|
||||
}
|
||||
|
||||
if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > maxFilterBlocks {
|
||||
return nil, errors.Errorf("maximum [from, to] blocks distance: %d", maxFilterBlocks)
|
||||
if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > blockLimit {
|
||||
return nil, errors.Errorf("maximum [from, to] blocks distance: %d", blockLimit)
|
||||
}
|
||||
|
||||
// check bounds
|
||||
@ -145,36 +144,37 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
|
||||
from := f.criteria.FromBlock.Int64()
|
||||
to := f.criteria.ToBlock.Int64()
|
||||
|
||||
blocks, err := f.backend.GetFilteredBlocks(from, to, f.bloomFilters, len(f.criteria.Addresses) > 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, height := range blocks {
|
||||
ethLogs, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
|
||||
for height := from; height <= to; height++ {
|
||||
bloom, err := f.backend.BlockBloom(&height)
|
||||
if err != nil {
|
||||
return logs, errors.Wrapf(err, "failed to fetch block by number %d", height)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, ethLog := range ethLogs {
|
||||
filtered := FilterLogs(ethLog, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics)
|
||||
logs = append(logs, filtered...)
|
||||
filtered, err := f.blockLogs(height, bloom)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to fetch block by number %d", height)
|
||||
}
|
||||
|
||||
// check logs limit
|
||||
if len(logs)+len(filtered) > logLimit {
|
||||
return nil, errors.Errorf("query returned more than %d results", logLimit)
|
||||
}
|
||||
logs = append(logs, filtered...)
|
||||
}
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
// blockLogs returns the logs matching the filter criteria within a single block.
|
||||
func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) {
|
||||
if !bloomFilter(header.Bloom, f.criteria.Addresses, f.criteria.Topics) {
|
||||
func (f *Filter) blockLogs(height int64, bloom ethtypes.Bloom) ([]*ethtypes.Log, error) {
|
||||
if !bloomFilter(bloom, f.criteria.Addresses, f.criteria.Topics) {
|
||||
return []*ethtypes.Log{}, nil
|
||||
}
|
||||
|
||||
// DANGER: do not call GetLogs(header.Hash())
|
||||
// eth header's hash doesn't match tm block hash
|
||||
logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(header.Number.Int64()))
|
||||
logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
|
||||
if err != nil {
|
||||
return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", header.Number.Int64())
|
||||
return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", height)
|
||||
}
|
||||
|
||||
unfiltered := make([]*ethtypes.Log, 0)
|
||||
|
@ -57,9 +57,10 @@ func includes(addresses []common.Address, a common.Address) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// https://github.com/ethereum/go-ethereum/blob/v1.10.14/eth/filters/filter.go#L321
|
||||
func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
|
||||
var included bool
|
||||
if len(addresses) > 0 {
|
||||
var included bool
|
||||
for _, addr := range addresses {
|
||||
if ethtypes.BloomLookup(bloom, addr) {
|
||||
included = true
|
||||
@ -72,15 +73,18 @@ func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]co
|
||||
}
|
||||
|
||||
for _, sub := range topics {
|
||||
included = len(sub) == 0 // empty rule set == wildcard
|
||||
included := len(sub) == 0 // empty rule set == wildcard
|
||||
for _, topic := range sub {
|
||||
if ethtypes.BloomLookup(bloom, topic) {
|
||||
included = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !included {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return included
|
||||
return true
|
||||
}
|
||||
|
||||
// returnHashes is a helper that will return an empty hash array case the given hash array is nil,
|
||||
|
@ -34,6 +34,10 @@ const (
|
||||
|
||||
DefaultFeeHistoryCap int32 = 100
|
||||
|
||||
DefaultLogsCap int32 = 10000
|
||||
|
||||
DefaultBlockRangeCap int32 = 10000
|
||||
|
||||
DefaultEVMTimeout = 5 * time.Second
|
||||
// default 1.0 eth
|
||||
DefaultTxFeeCap float64 = 1.0
|
||||
@ -78,6 +82,10 @@ type JSONRPCConfig struct {
|
||||
FeeHistoryCap int32 `mapstructure:"feehistory-cap"`
|
||||
// Enable defines if the EVM RPC server should be enabled.
|
||||
Enable bool `mapstructure:"enable"`
|
||||
// LogsCap defines the max number of results can be returned from single `eth_getLogs` query.
|
||||
LogsCap int32 `mapstructure:"logs-cap"`
|
||||
// BlockRangeCap defines the max block range allowed for `eth_getLogs` query.
|
||||
BlockRangeCap int32 `mapstructure:"block-range-cap"`
|
||||
}
|
||||
|
||||
// TLSConfig defines the certificate and matching private key for the server.
|
||||
@ -171,6 +179,8 @@ func DefaultJSONRPCConfig() *JSONRPCConfig {
|
||||
TxFeeCap: DefaultTxFeeCap,
|
||||
FilterCap: DefaultFilterCap,
|
||||
FeeHistoryCap: DefaultFeeHistoryCap,
|
||||
BlockRangeCap: DefaultBlockRangeCap,
|
||||
LogsCap: DefaultLogsCap,
|
||||
}
|
||||
}
|
||||
|
||||
@ -196,6 +206,14 @@ func (c JSONRPCConfig) Validate() error {
|
||||
return errors.New("JSON-RPC EVM timeout duration cannot be negative")
|
||||
}
|
||||
|
||||
if c.LogsCap < 0 {
|
||||
return errors.New("JSON-RPC logs cap cannot be negative")
|
||||
}
|
||||
|
||||
if c.BlockRangeCap < 0 {
|
||||
return errors.New("JSON-RPC block range cap cannot be negative")
|
||||
}
|
||||
|
||||
// TODO: validate APIs
|
||||
seenAPIs := make(map[string]bool)
|
||||
for _, api := range c.API {
|
||||
@ -253,6 +271,8 @@ func GetConfig(v *viper.Viper) Config {
|
||||
FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"),
|
||||
TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"),
|
||||
EVMTimeout: v.GetDuration("json-rpc.evm-timeout"),
|
||||
LogsCap: v.GetInt32("json-rpc.logs-cap"),
|
||||
BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"),
|
||||
},
|
||||
TLS: TLSConfig{
|
||||
CertificatePath: v.GetString("tls.certificate-path"),
|
||||
|
@ -47,6 +47,11 @@ filter-cap = {{ .JSONRPC.FilterCap }}
|
||||
# FeeHistoryCap sets the global cap for total number of blocks that can be fetched
|
||||
feehistory-cap = {{ .JSONRPC.FeeHistoryCap }}
|
||||
|
||||
# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
|
||||
logs-cap = {{ .JSONRPC.LogsCap }}
|
||||
|
||||
# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
|
||||
block-range-cap = {{ .JSONRPC.BlockRangeCap }}
|
||||
|
||||
###############################################################################
|
||||
### TLS Configuration ###
|
||||
|
@ -26,15 +26,17 @@ const (
|
||||
|
||||
// JSON-RPC flags
|
||||
const (
|
||||
JSONRPCEnable = "json-rpc.enable"
|
||||
JSONRPCAPI = "json-rpc.api"
|
||||
JSONRPCAddress = "json-rpc.address"
|
||||
JSONWsAddress = "json-rpc.ws-address"
|
||||
JSONRPCGasCap = "json-rpc.gas-cap"
|
||||
JSONRPCEVMTimeout = "json-rpc.evm-timeout"
|
||||
JSONRPCTxFeeCap = "json-rpc.txfee-cap"
|
||||
JSONRPCFilterCap = "json-rpc.filter-cap"
|
||||
JSONRPFeeHistoryCap = "json-rpc.feehistory-cap"
|
||||
JSONRPCEnable = "json-rpc.enable"
|
||||
JSONRPCAPI = "json-rpc.api"
|
||||
JSONRPCAddress = "json-rpc.address"
|
||||
JSONWsAddress = "json-rpc.ws-address"
|
||||
JSONRPCGasCap = "json-rpc.gas-cap"
|
||||
JSONRPCEVMTimeout = "json-rpc.evm-timeout"
|
||||
JSONRPCTxFeeCap = "json-rpc.txfee-cap"
|
||||
JSONRPCFilterCap = "json-rpc.filter-cap"
|
||||
JSONRPFeeHistoryCap = "json-rpc.feehistory-cap"
|
||||
JSONRPCLogsCap = "json-rpc.logs-cap"
|
||||
JSONRPCBlockRangeCap = "json-rpc.block-range-cap"
|
||||
)
|
||||
|
||||
// EVM flags
|
||||
|
@ -157,6 +157,8 @@ which accepts a path for the resulting pprof file.
|
||||
cmd.Flags().Float64(srvflags.JSONRPCTxFeeCap, config.DefaultTxFeeCap, "Sets a cap on transaction fee that can be sent via the RPC APIs (1 = default 1 photon)")
|
||||
cmd.Flags().Int32(srvflags.JSONRPCFilterCap, config.DefaultFilterCap, "Sets the global cap for total number of filters that can be created")
|
||||
cmd.Flags().Duration(srvflags.JSONRPCEVMTimeout, config.DefaultEVMTimeout, "Sets a timeout used for eth_call (0=infinite)")
|
||||
cmd.Flags().Int32(srvflags.JSONRPCLogsCap, config.DefaultLogsCap, "Sets the max number of results can be returned from single `eth_getLogs` query")
|
||||
cmd.Flags().Int32(srvflags.JSONRPCBlockRangeCap, config.DefaultBlockRangeCap, "Sets the max block range allowed for `eth_getLogs` query")
|
||||
|
||||
cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)")
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user