From cffceb53dbae2c1ca0987a5557ec33e4df7d581e Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 29 Oct 2020 22:07:39 -0500 Subject: [PATCH] optimize GetTransactionCount, GetBalance, and GetCode to use secondary indexes instead of operating through ethdb where we have to iterate down trie from root to leaf (multiple db lookups) to access account info --- pkg/eth/api.go | 52 ++++++++++++---------- pkg/eth/backend.go | 93 ++++++++++++++++++++++++++++++++++++++- pkg/eth/ipld_retriever.go | 22 ++++----- 3 files changed, 132 insertions(+), 35 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index f6e62018..3ad6401a 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -244,14 +244,9 @@ Transactions // GetTransactionCount returns the number of transactions the given address has sent for the given block number func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { - // Resolve block number and use its state to ask for the nonce - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - nonce := state.GetNonce(address) - err = state.Error() - if err == nil { - return (*hexutil.Uint64)(&nonce), nil - } + count, err := pea.localGetTransactionCount(ctx, address, blockNrOrHash) + if count != nil && err == nil { + return count, nil } if pea.rpc != nil { var num *hexutil.Uint64 @@ -262,6 +257,15 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common return nil, err } +func (pea *PublicEthAPI) localGetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { + account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash) + if err != nil { + return nil, err + } + nonce := hexutil.Uint64(account.Nonce) + return &nonce, nil +} + // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { @@ -401,6 +405,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common. } func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { + // TODO: this can be optimized for Postgres tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) if err != nil { return nil, err @@ -444,7 +449,7 @@ func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash co fields["status"] = hexutil.Uint(receipt.Status) } if receipt.Logs == nil { - fields["logs"] = [][]*types.Log{} + fields["logs"] = []*types.Log{} } // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation if receipt.ContractAddress != (common.Address{}) { @@ -492,6 +497,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery) } func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { + // TODO: this can be optimized away from using the old cid retriever and ipld fetcher interfaces // Convert FilterQuery into ReceiptFilter addrStrs := make([]string, len(crit.Addresses)) for i, addr := range crit.Addresses { @@ -588,13 +594,9 @@ State and Storage // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - balance := state.GetBalance(address) - err = state.Error() - if err == nil { - return (*hexutil.Big)(balance), nil - } + bal, err := pea.localGetBalance(ctx, address, blockNrOrHash) + if bal != nil && err == nil { + return bal, nil } if pea.rpc != nil { var res *hexutil.Big @@ -605,6 +607,14 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, return nil, err } +func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { + account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash) + if err != nil { + return nil, err + } + return (*hexutil.Big)(account.Balance), nil +} + // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. @@ -628,13 +638,9 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres // GetCode returns the code stored at the given address in the state for the given block number. func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - code := state.GetCode(address) - err = state.Error() - if err == nil { - return code, nil - } + code, err := pea.B.GetCodeByNumberOrHash(ctx, address, blockNrOrHash) + if code != nil && err == nil { + return code, nil } if pea.rpc != nil { var res hexutil.Bytes diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 5911a569..f711b0cf 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -19,6 +19,7 @@ package eth import ( "context" "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -30,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -38,6 +40,8 @@ import ( ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres" + shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -59,6 +63,12 @@ const ( WHERE blocks.key = transaction_cids.mh_key AND transaction_cids.header_id = header_cids.id AND transaction_cids.tx_hash = $1` + RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids + WHERE state_accounts.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND state_leaf_key = $1 + AND block_hash = $2` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) type Backend struct { @@ -417,7 +427,7 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece if err != nil { return nil, err } - rcts := make(types.Receipts, 0, len(receiptBytes)) + rcts := make(types.Receipts, len(receiptBytes)) for i, rctBytes := range receiptBytes { rct := new(types.Receipt) if err := rlp.DecodeBytes(rctBytes, rct); err != nil { @@ -512,6 +522,87 @@ func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.Sta return vm.NewEVM(c, state, b.Config.ChainConfig, b.Config.VmConfig), nil } +// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash +func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*state.Account, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetAccountByNumber(ctx, address, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetAccountByHash(ctx, address, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height +func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, number uint64) (*state.Account, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetAccountByHash(ctx, address, hash) +} + +// GetAccountByHash returns the account object for the provided address at the block with the provided hash +func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*state.Account, error) { + _, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash) + if err != nil { + return nil, err + } + acct := new(state.Account) + return acct, rlp.DecodeBytes(accountRlp, acct) +} + +// GetCodeByNumberOrHash returns the byte code for the contract deployed at the provided address at the block with the provided hash or block number +func (b *Backend) GetCodeByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) ([]byte, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetCodeByNumber(ctx, address, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetCodeByHash(ctx, address, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetCodeByNumber returns the byte code for the contract deployed at the provided address at the canonical block with the provided block number +func (b *Backend) GetCodeByNumber(ctx context.Context, address common.Address, number uint64) ([]byte, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetCodeByHash(ctx, address, hash) +} + +// GetCodeByHash returns the byte code for the contract deployed at the provided address at the block with the provided hash +func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, hash common.Hash) ([]byte, error) { + codeHash := make([]byte, 0) + leafKey := crypto.Keccak256Hash(address.Bytes()) + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + if err := tx.Get(&codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex()); err != nil { + return nil, err + } + mhKey, err := shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash)) + if err != nil { + return nil, err + } + code := make([]byte, 0) + err = tx.Get(&code, RetrieveCodeByMhKey, mhKey) + return code, err +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index b2800300..4d00712f 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -40,11 +40,11 @@ const ( RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveUnclesByBlockHashPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveUnclesByBlockNumberPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_number = $1` @@ -54,41 +54,41 @@ const ( RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveTransactionsByBlockHashPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveTransactionsByBlockNumberPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_number = $1` RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveReceiptsByTxHashesPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks + RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveReceiptsByBlockHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveReceiptsByBlockNumberPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_number = $1` - RetrieveReceiptByTxHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids + RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 AND block_hash = $2` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 @@ -333,7 +333,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr } // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number -// This can return multiple results if we have two versions of state in the database as the provided height +// This can return multiple results if we have two versions of state in the database at the provided height func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) { accountResults := make([]ipldResult, 0) leafKey := crypto.Keccak256Hash(address.Bytes())