optimize GetTransactionCount, GetBalance, and GetCode to use secondary indexes instead of operating through ethdb where we have to iterate down trie from root to leaf (multiple db lookups) to access account info

This commit is contained in:
Ian Norden 2020-10-29 22:07:39 -05:00
parent dc25ea7f87
commit cffceb53db
3 changed files with 132 additions and 35 deletions

View File

@ -244,14 +244,9 @@ Transactions
// GetTransactionCount returns the number of transactions the given address has sent for the given block number // GetTransactionCount returns the number of transactions the given address has sent for the given block number
func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) {
// Resolve block number and use its state to ask for the nonce count, err := pea.localGetTransactionCount(ctx, address, blockNrOrHash)
state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if count != nil && err == nil {
if state != nil && err == nil { return count, nil
nonce := state.GetNonce(address)
err = state.Error()
if err == nil {
return (*hexutil.Uint64)(&nonce), nil
}
} }
if pea.rpc != nil { if pea.rpc != nil {
var num *hexutil.Uint64 var num *hexutil.Uint64
@ -262,6 +257,15 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common
return nil, err return nil, err
} }
func (pea *PublicEthAPI) localGetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) {
account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash)
if err != nil {
return nil, err
}
nonce := hexutil.Uint64(account.Nonce)
return &nonce, nil
}
// GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number.
func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint {
if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
@ -401,6 +405,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.
} }
func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
// TODO: this can be optimized for Postgres
tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
@ -444,7 +449,7 @@ func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash co
fields["status"] = hexutil.Uint(receipt.Status) fields["status"] = hexutil.Uint(receipt.Status)
} }
if receipt.Logs == nil { if receipt.Logs == nil {
fields["logs"] = [][]*types.Log{} fields["logs"] = []*types.Log{}
} }
// If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation
if receipt.ContractAddress != (common.Address{}) { if receipt.ContractAddress != (common.Address{}) {
@ -492,6 +497,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
} }
func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) {
// TODO: this can be optimized away from using the old cid retriever and ipld fetcher interfaces
// Convert FilterQuery into ReceiptFilter // Convert FilterQuery into ReceiptFilter
addrStrs := make([]string, len(crit.Addresses)) addrStrs := make([]string, len(crit.Addresses))
for i, addr := range crit.Addresses { for i, addr := range crit.Addresses {
@ -588,13 +594,9 @@ State and Storage
// given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta
// block numbers are also allowed. // block numbers are also allowed.
func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) {
state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) bal, err := pea.localGetBalance(ctx, address, blockNrOrHash)
if state != nil && err == nil { if bal != nil && err == nil {
balance := state.GetBalance(address) return bal, nil
err = state.Error()
if err == nil {
return (*hexutil.Big)(balance), nil
}
} }
if pea.rpc != nil { if pea.rpc != nil {
var res *hexutil.Big var res *hexutil.Big
@ -605,6 +607,14 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address,
return nil, err return nil, err
} }
func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) {
account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash)
if err != nil {
return nil, err
}
return (*hexutil.Big)(account.Balance), nil
}
// GetStorageAt returns the storage from the state at the given address, key and // GetStorageAt returns the storage from the state at the given address, key and
// block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block
// numbers are also allowed. // numbers are also allowed.
@ -628,14 +638,10 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres
// GetCode returns the code stored at the given address in the state for the given block number. // GetCode returns the code stored at the given address in the state for the given block number.
func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) code, err := pea.B.GetCodeByNumberOrHash(ctx, address, blockNrOrHash)
if state != nil && err == nil { if code != nil && err == nil {
code := state.GetCode(address)
err = state.Error()
if err == nil {
return code, nil return code, nil
} }
}
if pea.rpc != nil { if pea.rpc != nil {
var res hexutil.Bytes var res hexutil.Bytes
if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil { if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {

View File

@ -19,6 +19,7 @@ package eth
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"math/big" "math/big"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
@ -30,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
@ -38,6 +40,8 @@ import (
ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb" ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/vulcanize/ipld-eth-server/pkg/shared" "github.com/vulcanize/ipld-eth-server/pkg/shared"
) )
@ -59,6 +63,12 @@ const (
WHERE blocks.key = transaction_cids.mh_key WHERE blocks.key = transaction_cids.mh_key
AND transaction_cids.header_id = header_cids.id AND transaction_cids.header_id = header_cids.id
AND transaction_cids.tx_hash = $1` AND transaction_cids.tx_hash = $1`
RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
WHERE state_accounts.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND state_leaf_key = $1
AND block_hash = $2`
RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
) )
type Backend struct { type Backend struct {
@ -417,7 +427,7 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece
if err != nil { if err != nil {
return nil, err return nil, err
} }
rcts := make(types.Receipts, 0, len(receiptBytes)) rcts := make(types.Receipts, len(receiptBytes))
for i, rctBytes := range receiptBytes { for i, rctBytes := range receiptBytes {
rct := new(types.Receipt) rct := new(types.Receipt)
if err := rlp.DecodeBytes(rctBytes, rct); err != nil { if err := rlp.DecodeBytes(rctBytes, rct); err != nil {
@ -512,6 +522,87 @@ func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.Sta
return vm.NewEVM(c, state, b.Config.ChainConfig, b.Config.VmConfig), nil return vm.NewEVM(c, state, b.Config.ChainConfig, b.Config.VmConfig), nil
} }
// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*state.Account, error) {
if blockNr, ok := blockNrOrHash.Number(); ok {
return b.GetAccountByNumber(ctx, address, uint64(blockNr.Int64()))
}
if hash, ok := blockNrOrHash.Hash(); ok {
return b.GetAccountByHash(ctx, address, hash)
}
return nil, errors.New("invalid arguments; neither block nor hash specified")
}
// GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height
func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, number uint64) (*state.Account, error) {
hash := b.GetCanonicalHash(number)
if hash == (common.Hash{}) {
return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
}
return b.GetAccountByHash(ctx, address, hash)
}
// GetAccountByHash returns the account object for the provided address at the block with the provided hash
func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*state.Account, error) {
_, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash)
if err != nil {
return nil, err
}
acct := new(state.Account)
return acct, rlp.DecodeBytes(accountRlp, acct)
}
// GetCodeByNumberOrHash returns the byte code for the contract deployed at the provided address at the block with the provided hash or block number
func (b *Backend) GetCodeByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) ([]byte, error) {
if blockNr, ok := blockNrOrHash.Number(); ok {
return b.GetCodeByNumber(ctx, address, uint64(blockNr.Int64()))
}
if hash, ok := blockNrOrHash.Hash(); ok {
return b.GetCodeByHash(ctx, address, hash)
}
return nil, errors.New("invalid arguments; neither block nor hash specified")
}
// GetCodeByNumber returns the byte code for the contract deployed at the provided address at the canonical block with the provided block number
func (b *Backend) GetCodeByNumber(ctx context.Context, address common.Address, number uint64) ([]byte, error) {
hash := b.GetCanonicalHash(number)
if hash == (common.Hash{}) {
return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
}
return b.GetCodeByHash(ctx, address, hash)
}
// GetCodeByHash returns the byte code for the contract deployed at the provided address at the block with the provided hash
func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, hash common.Hash) ([]byte, error) {
codeHash := make([]byte, 0)
leafKey := crypto.Keccak256Hash(address.Bytes())
// Begin tx
tx, err := b.DB.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
if err := tx.Get(&codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex()); err != nil {
return nil, err
}
mhKey, err := shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
if err != nil {
return nil, err
}
code := make([]byte, 0)
err = tx.Get(&code, RetrieveCodeByMhKey, mhKey)
return code, err
}
// Engine satisfied the ChainContext interface // Engine satisfied the ChainContext interface
func (b *Backend) Engine() consensus.Engine { func (b *Backend) Engine() consensus.Engine {
// TODO: we need to support more than just ethash based engines // TODO: we need to support more than just ethash based engines

View File

@ -40,11 +40,11 @@ const (
RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
WHERE block_hash = ANY($1::VARCHAR(66)[])` WHERE block_hash = ANY($1::VARCHAR(66)[])`
RetrieveUnclesByBlockHashPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks
WHERE uncle_cids.header_id = header_cids.id WHERE uncle_cids.header_id = header_cids.id
AND uncle_cids.mh_key = blocks.key AND uncle_cids.mh_key = blocks.key
AND block_hash = $1` AND block_hash = $1`
RetrieveUnclesByBlockNumberPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks
WHERE uncle_cids.header_id = header_cids.id WHERE uncle_cids.header_id = header_cids.id
AND uncle_cids.mh_key = blocks.key AND uncle_cids.mh_key = blocks.key
AND block_number = $1` AND block_number = $1`
@ -54,41 +54,41 @@ const (
RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = ANY($1::VARCHAR(66)[])` WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveTransactionsByBlockHashPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks
WHERE transaction_cids.header_id = header_cids.id WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.mh_key = blocks.key AND transaction_cids.mh_key = blocks.key
AND block_hash = $1` AND block_hash = $1`
RetrieveTransactionsByBlockNumberPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks
WHERE transaction_cids.header_id = header_cids.id WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.mh_key = blocks.key AND transaction_cids.mh_key = blocks.key
AND block_number = $1` AND block_number = $1`
RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = $1` WHERE tx_hash = $1`
RetrieveReceiptsByTxHashesPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks
WHERE receipt_cids.mh_key = blocks.key WHERE receipt_cids.mh_key = blocks.key
AND receipt_cids.tx_id = transaction_cids.id AND receipt_cids.tx_id = transaction_cids.id
AND tx_hash = ANY($1::VARCHAR(66)[])` AND tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveReceiptsByBlockHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE receipt_cids.tx_id = transaction_cids.id WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id AND transaction_cids.header_id = header_cids.id
AND receipt_cids.mh_key = blocks.key AND receipt_cids.mh_key = blocks.key
AND block_hash = $1` AND block_hash = $1`
RetrieveReceiptsByBlockNumberPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE receipt_cids.tx_id = transaction_cids.id WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id AND transaction_cids.header_id = header_cids.id
AND receipt_cids.mh_key = blocks.key AND receipt_cids.mh_key = blocks.key
AND block_number = $1` AND block_number = $1`
RetrieveReceiptByTxHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids
WHERE receipt_cids.mh_key = blocks.key WHERE receipt_cids.mh_key = blocks.key
AND receipt_cids.tx_id = transaction_cids.id AND receipt_cids.tx_id = transaction_cids.id
AND tx_hash = $1` AND tx_hash = $1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks
WHERE state_cids.header_id = header_cids.id WHERE state_cids.header_id = header_cids.id
AND state_cids.mh_key = blocks.key AND state_cids.mh_key = blocks.key
AND state_leaf_key = $1 AND state_leaf_key = $1
AND block_hash = $2` AND block_hash = $2`
RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks
WHERE state_cids.header_id = header_cids.id WHERE state_cids.header_id = header_cids.id
AND state_cids.mh_key = blocks.key AND state_cids.mh_key = blocks.key
AND state_leaf_key = $1 AND state_leaf_key = $1
@ -333,7 +333,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
} }
// RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
// This can return multiple results if we have two versions of state in the database as the provided height // This can return multiple results if we have two versions of state in the database at the provided height
func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) { func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) {
accountResults := make([]ipldResult, 0) accountResults := make([]ipldResult, 0)
leafKey := crypto.Keccak256Hash(address.Bytes()) leafKey := crypto.Keccak256Hash(address.Bytes())