From b5d57b6afc1e39afacdc829de3dcf095c47ea6de Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Tue, 27 Oct 2020 22:04:19 -0500 Subject: [PATCH 01/10] the remaining, currently supportable, eth_* endpoints --- pkg/eth/api.go | 476 +++++++++++++++++++++++++++++++++------ pkg/eth/backend.go | 17 +- pkg/eth/backend_utils.go | 11 + 3 files changed, 424 insertions(+), 80 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 9a792c0b..ec62432e 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -28,9 +28,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/sirupsen/logrus" @@ -66,12 +69,322 @@ func NewPublicEthAPI(b *Backend, client *rpc.Client) *PublicEthAPI { } } +/* + +Headers and blocks + +*/ + +// GetHeaderByNumber returns the requested canonical block header. +// * When blockNr is -1 the chain head is returned. +// * We cannot support pending block calls since we do not have an active miner +func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { + header, err := pea.B.HeaderByNumber(ctx, number) + if header != nil && err == nil { + return pea.rpcMarshalHeader(header) + } + if pea.ethClient != nil { + if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil { + return pea.rpcMarshalHeader(header) + } + } + return nil, err +} + +// GetHeaderByHash returns the requested header by hash. +func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { + header, err := pea.B.HeaderByHash(ctx, hash) + if header != nil && err == nil { + if res, err := pea.rpcMarshalHeader(header); err != nil { + return res + } + } + if pea.ethClient != nil { + if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil { + if res, err := pea.rpcMarshalHeader(header); err != nil { + return res + } + } + } + return nil +} + +// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field +func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) { + fields := RPCMarshalHeader(header) + td, err := pea.B.GetTd(header.Hash()) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(td) + return fields, nil +} + // BlockNumber returns the block number of the chain head. func (pea *PublicEthAPI) BlockNumber() hexutil.Uint64 { number, _ := pea.B.Retriever.RetrieveLastBlockNumber() return hexutil.Uint64(number) } +// GetBlockByNumber returns the requested canonical block. +// * When blockNr is -1 the chain head is returned. +// * We cannot support pending block calls since we do not have an active miner +// * When fullTx is true all transactions in the block are returned, otherwise +// only the transaction hash is returned. +func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { + block, err := pea.B.BlockByNumber(ctx, number) + if block != nil && err == nil { + return pea.rpcMarshalBlock(block, true, fullTx) + } + if pea.ethClient != nil { + if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil { + return pea.rpcMarshalBlock(block, true, fullTx) + } + } + return nil, err +} + +// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full +// detail, otherwise only the transaction hash is returned. +func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { + block, err := pea.B.BlockByHash(ctx, hash) + if block != nil && err == nil { + return pea.rpcMarshalBlock(block, true, fullTx) + } + if pea.ethClient != nil { + if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil { + return pea.rpcMarshalBlock(block, true, fullTx) + } + } + return nil, err +} + +/* + +Uncles + +*/ + +// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true +// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. +func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { + block, err := pea.B.BlockByNumber(ctx, blockNr) + if block != nil { + uncles := block.Uncles() + if index >= hexutil.Uint(len(uncles)) { + logrus.Debugf("uncle with index %s request at block number %d was not found", index.String(), blockNr.Int64()) + return nil, nil + } + block = types.NewBlockWithHeader(uncles[index]) + return pea.rpcMarshalBlock(block, false, false) + } + return nil, err +} + +// GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true +// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. +func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { + block, err := pea.B.BlockByHash(ctx, blockHash) + if block != nil { + uncles := block.Uncles() + if index >= hexutil.Uint(len(uncles)) { + logrus.Debugf("uncle with index %s request at block hash %s was not found", index.String(), blockHash.Hex()) + return nil, nil + } + block = types.NewBlockWithHeader(uncles[index]) + return pea.rpcMarshalBlock(block, false, false) + } + return nil, err +} + +// GetUncleCountByBlockNumber returns number of uncles in the block for the given block number +func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { + if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { + n := hexutil.Uint(len(block.Uncles())) + return &n + } + return nil +} + +// GetUncleCountByBlockHash returns number of uncles in the block for the given block hash +func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { + if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { + n := hexutil.Uint(len(block.Uncles())) + return &n + } + return nil +} + +/* + +Transactions + +*/ + +// GetTransactionCount returns the number of transactions the given address has sent for the given block number +func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { + // Resolve block number and use its state to ask for the nonce + state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + nonce := state.GetNonce(address) + return (*hexutil.Uint64)(&nonce), state.Error() +} + +// GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. +func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { + if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { + n := hexutil.Uint(len(block.Transactions())) + return &n + } + return nil +} + +// GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. +func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { + if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { + n := hexutil.Uint(len(block.Transactions())) + return &n + } + return nil +} + +// GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. +func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { + if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index)) + } + return nil +} + +// GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. +func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { + if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index)) + } + return nil +} + +// GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index. +func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes { + if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { + return newRPCRawTransactionFromBlockIndex(block, uint64(index)) + } + return nil +} + +// GetRawTransactionByBlockHashAndIndex returns the bytes of the transaction for the given block hash and index. +func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes { + if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { + return newRPCRawTransactionFromBlockIndex(block, uint64(index)) + } + return nil +} + +// GetTransactionByHash returns the transaction for the given hash +// eth ipld-eth-server cannot currently handle pending/tx_pool txs +func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { + tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) + if tx != nil && err == nil { + return NewRPCTransaction(tx, blockHash, blockNumber, index), nil + } + if pea.rpc != nil { + if tx, err := pea.remoteGetTransactionByHash(ctx, hash); tx != nil && err == nil { + return tx, nil + } + } + return nil, err +} + +func (pea *PublicEthAPI) remoteGetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { + var tx *RPCTransaction + if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); err != nil { + return nil, err + } + return tx, nil +} + +// GetRawTransactionByHash returns the bytes of the transaction for the given hash. +func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { + // Retrieve a finalized transaction, or a pooled otherwise + tx, _, _, _, err := pea.B.GetTransaction(ctx, hash) + if tx != nil && err == nil { + return rlp.EncodeToBytes(tx) + } + if pea.rpc != nil { + if tx, err := pea.remoteGetRawTransactionByHash(ctx, hash); tx != nil && err == nil { + return tx, nil + } + } + return nil, err +} + +func (pea *PublicEthAPI) remoteGetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { + var tx hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); err != nil { + return nil, err + } + return tx, nil +} + +/* + +Receipts and Logs + +*/ + +// GetTransactionReceipt returns the transaction receipt for the given transaction hash. +func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { + tx, blockHash, blockNumber, index := rawdb.ReadTransaction(pea.B.ChainDb(), hash) + if tx == nil { + return nil, nil + } + receipts, err := pea.B.GetReceipts(ctx, blockHash) + if err != nil { + return nil, err + } + if len(receipts) <= int(index) { + return nil, nil + } + receipt := receipts[index] + + var signer types.Signer = types.FrontierSigner{} + if tx.Protected() { + signer = types.NewEIP155Signer(tx.ChainId()) + } + from, _ := types.Sender(signer, tx) + + fields := map[string]interface{}{ + "blockHash": blockHash, + "blockNumber": hexutil.Uint64(blockNumber), + "transactionHash": hash, + "transactionIndex": hexutil.Uint64(index), + "from": from, + "to": tx.To(), + "gasUsed": hexutil.Uint64(receipt.GasUsed), + "cumulativeGasUsed": hexutil.Uint64(receipt.CumulativeGasUsed), + "contractAddress": nil, + "logs": receipt.Logs, + "logsBloom": receipt.Bloom, + } + + // Assign receipt status or post state. + if len(receipt.PostState) > 0 { + fields["root"] = hexutil.Bytes(receipt.PostState) + } else { + fields["status"] = hexutil.Uint(receipt.Status) + } + if receipt.Logs == nil { + fields["logs"] = [][]*types.Log{} + } + // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation + if receipt.ContractAddress != (common.Address{}) { + fields["contractAddress"] = receipt.ContractAddress + } + return fields, nil +} + // GetLogs returns logs matching the given argument that are stored within the state. // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs @@ -175,76 +488,109 @@ func (pea *PublicEthAPI) getLogs(ctx context.Context, crit ethereum.FilterQuery) return logs, err // need to return err variable so that we return the err = tx.Commit() assignment in the defer } -// GetHeaderByNumber returns the requested canonical block header. -// * When blockNr is -1 the chain head is returned. -// * We cannot support pending block calls since we do not have an active miner -func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { - header, err := pea.B.HeaderByNumber(ctx, number) - if header != nil && err == nil { - return pea.rpcMarshalHeader(header) - } - if pea.ethClient != nil { - if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil { - return pea.rpcMarshalHeader(header) - } - } - return nil, err -} +/* -// GetBlockByNumber returns the requested canonical block. -// * When blockNr is -1 the chain head is returned. -// * We cannot support pending block calls since we do not have an active miner -// * When fullTx is true all transactions in the block are returned, otherwise -// only the transaction hash is returned. -func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { - block, err := pea.B.BlockByNumber(ctx, number) - if block != nil && err == nil { - return pea.rpcMarshalBlock(block, true, fullTx) - } - if pea.ethClient != nil { - if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil { - return pea.rpcMarshalBlock(block, true, fullTx) - } - } - return nil, err -} +State and Storage -// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full -// detail, otherwise only the transaction hash is returned. -func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { - block, err := pea.B.BlockByHash(ctx, hash) - if block != nil && err == nil { - return pea.rpcMarshalBlock(block, true, fullTx) - } - if pea.ethClient != nil { - if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil { - return pea.rpcMarshalBlock(block, true, fullTx) - } - } - return nil, err -} +*/ -// GetTransactionByHash returns the transaction for the given hash -// eth ipld-eth-server cannot currently handle pending/tx_pool txs -func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { - tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) - if tx != nil && err == nil { - return NewRPCTransaction(tx, blockHash, blockNumber, index), nil - } - if pea.rpc != nil { - if tx, err := pea.remoteGetTransactionByHash(ctx, hash); tx != nil && err == nil { - return tx, nil - } - } - return nil, err -} - -func (pea *PublicEthAPI) remoteGetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { - var tx *RPCTransaction - if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); err != nil { +// GetBalance returns the amount of wei for the given address in the state of the +// given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta +// block numbers are also allowed. +func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { + state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { return nil, err } - return tx, nil + return (*hexutil.Big)(state.GetBalance(address)), state.Error() +} + +// GetStorageAt returns the storage from the state at the given address, key and +// block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block +// numbers are also allowed. +func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + res := state.GetState(address, common.HexToHash(key)) + return res[:], state.Error() +} + +// GetCode returns the code stored at the given address in the state for the given block number. +func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + code := state.GetCode(address) + return code, state.Error() +} + +// Result structs for GetProof +type AccountResult struct { + Address common.Address `json:"address"` + AccountProof []string `json:"accountProof"` + Balance *hexutil.Big `json:"balance"` + CodeHash common.Hash `json:"codeHash"` + Nonce hexutil.Uint64 `json:"nonce"` + StorageHash common.Hash `json:"storageHash"` + StorageProof []StorageResult `json:"storageProof"` +} +type StorageResult struct { + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` +} + +// GetProof returns the Merkle-proof for a given account and optionally some storage keys. +func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { + state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + + storageTrie := state.StorageTrie(address) + storageHash := types.EmptyRootHash + codeHash := state.GetCodeHash(address) + storageProof := make([]StorageResult, len(storageKeys)) + + // if we have a storageTrie, (which means the account exists), we can update the storagehash + if storageTrie != nil { + storageHash = storageTrie.Hash() + } else { + // no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray. + codeHash = crypto.Keccak256Hash(nil) + } + + // create the proof for the storageKeys + for i, key := range storageKeys { + if storageTrie != nil { + proof, storageError := state.GetStorageProof(address, common.HexToHash(key)) + if storageError != nil { + return nil, storageError + } + storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)} + } else { + storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + } + } + + // create the accountProof + accountProof, proofErr := state.GetProof(address) + if proofErr != nil { + return nil, proofErr + } + + return &AccountResult{ + Address: address, + AccountProof: common.ToHexArray(accountProof), + Balance: (*hexutil.Big)(state.GetBalance(address)), + CodeHash: codeHash, + Nonce: hexutil.Uint64(state.GetNonce(address)), + StorageHash: storageHash, + StorageProof: storageProof, + }, state.Error() } // Call executes the given transaction on the state for the given block number. diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 4e5ff088..fee552e0 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -21,19 +21,18 @@ import ( "errors" "math/big" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" @@ -167,18 +166,6 @@ func (b *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.Bl return nil, errors.New("invalid arguments; neither block nor hash specified") } -// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicEthAPI`. -func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) { - fields := RPCMarshalHeader(header) - td, err := pea.B.GetTd(header.Hash()) - if err != nil { - return nil, err - } - fields["totalDifficulty"] = (*hexutil.Big)(td) - return fields, nil -} - // GetTd gets the total difficulty at the given block hash func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) { var tdStr string diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index 9d7277eb..84f3dd91 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + "github.com/vulcanize/ipld-eth-indexer/pkg/ipfs" ) @@ -127,6 +128,16 @@ func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber return result } +// newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index. +func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.Bytes { + txs := b.Transactions() + if index >= uint64(len(txs)) { + return nil + } + blob, _ := rlp.EncodeToBytes(txs[index]) + return blob +} + // newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation. func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction { txs := b.Transactions() From 1d4abcb69b8ff6cea53c22fe9941c19ec5009dc3 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 28 Oct 2020 08:22:57 -0500 Subject: [PATCH 02/10] finish cache miss forwarding for new endpoints --- pkg/eth/api.go | 318 ++++++++++++++++++++++++--------------- pkg/eth/backend.go | 16 -- pkg/eth/backend_utils.go | 124 +++++++++++++++ pkg/eth/types.go | 60 ++++++++ 4 files changed, 380 insertions(+), 138 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index ec62432e..81a5606c 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" @@ -159,17 +158,11 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f return nil, err } -/* - -Uncles - -*/ - // GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true // all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { block, err := pea.B.BlockByNumber(ctx, blockNr) - if block != nil { + if block != nil && err == nil { uncles := block.Uncles() if index >= hexutil.Uint(len(uncles)) { logrus.Debugf("uncle with index %s request at block number %d was not found", index.String(), blockNr.Int64()) @@ -178,6 +171,11 @@ func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, bloc block = types.NewBlockWithHeader(uncles[index]) return pea.rpcMarshalBlock(block, false, false) } + if pea.rpc != nil { + if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil { + return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false) + } + } return nil, err } @@ -194,24 +192,41 @@ func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockH block = types.NewBlockWithHeader(uncles[index]) return pea.rpcMarshalBlock(block, false, false) } + if pea.rpc != nil { + if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil { + return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false) + } + } return nil, err } // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { - if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { + if block, err := pea.B.BlockByNumber(ctx, blockNr); block != nil && err == nil { n := hexutil.Uint(len(block.Uncles())) return &n } + if pea.rpc != nil { + var num *hexutil.Uint + if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil { + return num + } + } return nil } // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { - if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { + if block, err := pea.B.BlockByHash(ctx, blockHash); block != nil && err == nil { n := hexutil.Uint(len(block.Uncles())) return &n } + if pea.rpc != nil { + var num *hexutil.Uint + if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil { + return num + } + } return nil } @@ -225,11 +240,20 @@ Transactions func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { // Resolve block number and use its state to ask for the nonce state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err + if state != nil && err == nil { + nonce := state.GetNonce(address) + err = state.Error() + if err == nil { + return (*hexutil.Uint64)(&nonce), nil + } } - nonce := state.GetNonce(address) - return (*hexutil.Uint64)(&nonce), state.Error() + if pea.rpc != nil { + var num *hexutil.Uint64 + if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil { + return num, nil + } + } + return nil, err } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. @@ -238,6 +262,12 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, b n := hexutil.Uint(len(block.Transactions())) return &n } + if pea.rpc != nil { + var num *hexutil.Uint + if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil { + return num + } + } return nil } @@ -247,6 +277,12 @@ func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blo n := hexutil.Uint(len(block.Transactions())) return &n } + if pea.rpc != nil { + var num *hexutil.Uint + if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil { + return num + } + } return nil } @@ -255,6 +291,12 @@ func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index)) } + if pea.rpc != nil { + var tx *RPCTransaction + if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil { + return tx + } + } return nil } @@ -263,6 +305,12 @@ func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index)) } + if pea.rpc != nil { + var tx *RPCTransaction + if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil { + return tx + } + } return nil } @@ -271,6 +319,12 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Cont if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } + if pea.rpc != nil { + var tx hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil { + return tx + } + } return nil } @@ -279,6 +333,12 @@ func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Contex if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } + if pea.rpc != nil { + var tx hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil { + return tx + } + } return nil } @@ -290,21 +350,14 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H return NewRPCTransaction(tx, blockHash, blockNumber, index), nil } if pea.rpc != nil { - if tx, err := pea.remoteGetTransactionByHash(ctx, hash); tx != nil && err == nil { + var tx *RPCTransaction + if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil { return tx, nil } } return nil, err } -func (pea *PublicEthAPI) remoteGetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { - var tx *RPCTransaction - if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); err != nil { - return nil, err - } - return tx, nil -} - // GetRawTransactionByHash returns the bytes of the transaction for the given hash. func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise @@ -313,21 +366,14 @@ func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash commo return rlp.EncodeToBytes(tx) } if pea.rpc != nil { - if tx, err := pea.remoteGetRawTransactionByHash(ctx, hash); tx != nil && err == nil { + var tx hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil { return tx, nil } } return nil, err } -func (pea *PublicEthAPI) remoteGetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { - var tx hexutil.Bytes - if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); err != nil { - return nil, err - } - return tx, nil -} - /* Receipts and Logs @@ -336,12 +382,21 @@ Receipts and Logs // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { - tx, blockHash, blockNumber, index := rawdb.ReadTransaction(pea.B.ChainDb(), hash) + tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) + if err != nil { + if rct := pea.remoteGetTransactionReceipt(ctx, hash); rct != nil { + return rct, nil + } + return nil, err + } if tx == nil { return nil, nil } receipts, err := pea.B.GetReceipts(ctx, blockHash) if err != nil { + if rct := pea.remoteGetTransactionReceipt(ctx, hash); rct != nil { + return rct, nil + } return nil, err } if len(receipts) <= int(index) { @@ -385,23 +440,47 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common. return fields, nil } +func (pea *PublicEthAPI) remoteGetTransactionReceipt(ctx context.Context, hash common.Hash) map[string]interface{} { + if pea.rpc != nil { + var rct *RPCReceipt + if err := pea.rpc.CallContext(ctx, &rct, "eth_getTransactionReceipt", hash); rct != nil && err == nil { + return map[string]interface{}{ + "blockHash": rct.BlockHash, + "blockNumber": rct.BlockNumber, + "transactionHash": rct.TransactionHash, + "transactionIndex": rct.TransactionIndex, + "from": rct.From, + "to": rct.To, + "gasUsed": rct.GasUsed, + "cumulativeGasUsed": rct.CumulativeGsUsed, + "contractAddress": rct.ContractAddress, + "logs": rct.Logs, + "logsBloom": rct.Bloom, + "root": rct.Root, + "status": rct.Status, + } + } + } + return nil +} + // GetLogs returns logs matching the given argument that are stored within the state. // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { - logs, err := pea.getLogs(ctx, crit) + logs, err := pea.localGetLogs(ctx, crit) if err != nil && pea.rpc != nil { if arg, err := toFilterArg(crit); err == nil { - var result []*types.Log - if err := pea.rpc.CallContext(ctx, &result, "eth_getLogs", arg); err == nil { - return result, nil + var res []*types.Log + if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil { + return res, nil } } } return logs, err } -func (pea *PublicEthAPI) getLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { +func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { // Convert FilterQuery into ReceiptFilter addrStrs := make([]string, len(crit.Addresses)) for i, addr := range crit.Addresses { @@ -499,10 +578,20 @@ State and Storage // block numbers are also allowed. func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err + if state != nil && err == nil { + balance := state.GetBalance(address) + err = state.Error() + if err == nil { + return (*hexutil.Big)(balance), nil + } } - return (*hexutil.Big)(state.GetBalance(address)), state.Error() + if pea.rpc != nil { + var res *hexutil.Big + if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil { + return res, nil + } + } + return nil, err } // GetStorageAt returns the storage from the state at the given address, key and @@ -510,41 +599,57 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, // numbers are also allowed. func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err + if state != nil && err == nil { + res := state.GetState(address, common.HexToHash(key)) + err = state.Error() + if err == nil { + return res[:], nil + } } - res := state.GetState(address, common.HexToHash(key)) - return res[:], state.Error() + if pea.rpc != nil { + var res hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil { + return res, nil + } + } + return nil, err } // GetCode returns the code stored at the given address in the state for the given block number. func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err + if state != nil && err == nil { + code := state.GetCode(address) + err = state.Error() + if err == nil { + return code, nil + } } - code := state.GetCode(address) - return code, state.Error() -} - -// Result structs for GetProof -type AccountResult struct { - Address common.Address `json:"address"` - AccountProof []string `json:"accountProof"` - Balance *hexutil.Big `json:"balance"` - CodeHash common.Hash `json:"codeHash"` - Nonce hexutil.Uint64 `json:"nonce"` - StorageHash common.Hash `json:"storageHash"` - StorageProof []StorageResult `json:"storageProof"` -} -type StorageResult struct { - Key string `json:"key"` - Value *hexutil.Big `json:"value"` - Proof []string `json:"proof"` + if pea.rpc != nil { + var res hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil { + return res, nil + } + } + return nil, err } // GetProof returns the Merkle-proof for a given account and optionally some storage keys. func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { + proof, err := pea.localGetProof(ctx, address, storageKeys, blockNrOrHash) + if proof != nil && err == nil { + return proof, nil + } + if pea.rpc != nil { + var res *AccountResult + if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil { + return res, nil + } + } + return nil, err +} + +func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -604,48 +709,17 @@ func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash if overrides != nil { accounts = *overrides } - result, _, failed, err := DoCall(ctx, pea.B, args, blockNrOrHash, accounts, 5*time.Second, pea.B.Config.RPCGasCap) + res, _, failed, err := DoCall(ctx, pea.B, args, blockNrOrHash, accounts, 5*time.Second, pea.B.Config.RPCGasCap) if (failed || err != nil) && pea.rpc != nil { - if res, err := pea.remoteCall(ctx, args, blockNrOrHash, overrides); res != nil && err == nil { - return res, nil + var hex hexutil.Bytes + if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil { + return hex, nil } } if failed && err == nil { return nil, errors.New("eth_call failed without error") } - return (hexutil.Bytes)(result), err -} - -func (pea *PublicEthAPI) remoteCall(ctx context.Context, msg CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *map[common.Address]account) (hexutil.Bytes, error) { - var hex hexutil.Bytes - if err := pea.rpc.CallContext(ctx, &hex, "eth_call", msg, blockNrOrHash, overrides); err != nil { - return nil, err - } - return hex, nil -} - -// CallArgs represents the arguments for a call. -type CallArgs struct { - From *common.Address `json:"from"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - Value *hexutil.Big `json:"value"` - Data *hexutil.Bytes `json:"data"` -} - -// account indicates the overriding fields of account during the execution of -// a message call. -// Note, state and stateDiff can't be specified at the same time. If state is -// set, message execution will only use the data in the given state. Otherwise -// if statDiff is set, all diff will be applied first and then execute the call -// message. -type account struct { - Nonce *hexutil.Uint64 `json:"nonce"` - Code *hexutil.Bytes `json:"code"` - Balance **hexutil.Big `json:"balance"` - State *map[common.Hash]common.Hash `json:"state"` - StateDiff *map[common.Hash]common.Hash `json:"stateDiff"` + return (hexutil.Bytes)(res), err } func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]account, timeout time.Duration, globalGasCap *big.Int) ([]byte, uint64, bool, error) { @@ -755,30 +829,30 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl return res, gas, failed, err } -func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { - arg := map[string]interface{}{ - "address": q.Addresses, - "topics": q.Topics, +// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field +func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields, err := RPCMarshalBlock(b, inclTx, fullTx) + if err != nil { + return nil, err } - if q.BlockHash != nil { - arg["blockHash"] = *q.BlockHash - if q.FromBlock != nil || q.ToBlock != nil { - return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") - } - } else { - if q.FromBlock == nil { - arg["fromBlock"] = "0x0" - } else { - arg["fromBlock"] = toBlockNumArg(q.FromBlock) - } - arg["toBlock"] = toBlockNumArg(q.ToBlock) + td, err := pea.B.GetTd(b.Hash()) + if err != nil { + return nil, err } - return arg, nil + fields["totalDifficulty"] = (*hexutil.Big)(td) + return fields, err } -func toBlockNumArg(number *big.Int) string { - if number == nil { - return "latest" +// rpcMarshalBlockWithUncleHashes uses the generalized output filler, then adds the total difficulty field +func (pea *PublicEthAPI) rpcMarshalBlockWithUncleHashes(b *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields, err := RPCMarshalBlockWithUncleHashes(b, uncleHashes, inclTx, fullTx) + if err != nil { + return nil, err } - return hexutil.EncodeBig(number) + td, err := pea.B.GetTd(b.Hash()) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(td) + return fields, err } diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index fee552e0..5911a569 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -22,7 +22,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -446,21 +445,6 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log return logs, nil } -// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicBlockchainAPI`. -func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx) - if err != nil { - return nil, err - } - td, err := pea.B.GetTd(b.Hash()) - if err != nil { - return nil, err - } - fields["totalDifficulty"] = (*hexutil.Big)(td) - return fields, err -} - // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { if blockNr, ok := blockNrOrHash.Number(); ok { diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index 84f3dd91..98b1e33c 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -17,8 +17,14 @@ package eth import ( + "context" + "encoding/json" + "fmt" "math/big" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -87,6 +93,35 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i return fields, nil } +// RPCMarshalBlockWithUncleHashes marshals the block with the provided uncle hashes +func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header()) + fields["size"] = hexutil.Uint64(block.Size()) + + if inclTx { + formatTx := func(tx *types.Transaction) (interface{}, error) { + return tx.Hash(), nil + } + if fullTx { + formatTx = func(tx *types.Transaction) (interface{}, error) { + return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil + } + } + txs := block.Transactions() + transactions := make([]interface{}, len(txs)) + var err error + for i, tx := range txs { + if transactions[i], err = formatTx(tx); err != nil { + return nil, err + } + } + fields["transactions"] = transactions + } + fields["uncles"] = uncleHashes + + return fields, nil +} + // NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation. func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction { for idx, tx := range b.Transactions() { @@ -128,6 +163,67 @@ func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber return result } +type rpcBlock struct { + Hash common.Hash `json:"hash"` + Transactions []rpcTransaction `json:"transactions"` + UncleHashes []common.Hash `json:"uncles"` +} + +type rpcTransaction struct { + tx *types.Transaction + txExtraInfo +} + +type txExtraInfo struct { + BlockNumber *string `json:"blockNumber,omitempty"` + BlockHash *common.Hash `json:"blockHash,omitempty"` + From *common.Address `json:"from,omitempty"` +} + +func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error { + if err := json.Unmarshal(msg, &tx.tx); err != nil { + return err + } + return json.Unmarshal(msg, &tx.txExtraInfo) +} + +func getBlockAndUncleHashes(cli *rpc.Client, ctx context.Context, method string, args ...interface{}) (*types.Block, []common.Hash, error) { + var raw json.RawMessage + err := cli.CallContext(ctx, &raw, method, args...) + if err != nil { + return nil, nil, err + } else if len(raw) == 0 { + return nil, nil, ethereum.NotFound + } + // Decode header and transactions. + var head *types.Header + var body rpcBlock + if err := json.Unmarshal(raw, &head); err != nil { + return nil, nil, err + } + if err := json.Unmarshal(raw, &body); err != nil { + return nil, nil, err + } + // Quick-verify transaction and uncle lists. This mostly helps with debugging the server. + if head.UncleHash == types.EmptyUncleHash && len(body.UncleHashes) > 0 { + return nil, nil, fmt.Errorf("server returned non-empty uncle list but block header indicates no uncles") + } + if head.UncleHash != types.EmptyUncleHash && len(body.UncleHashes) == 0 { + return nil, nil, fmt.Errorf("server returned empty uncle list but block header indicates uncles") + } + if head.TxHash == types.EmptyRootHash && len(body.Transactions) > 0 { + return nil, nil, fmt.Errorf("server returned non-empty transaction list but block header indicates no transactions") + } + if head.TxHash != types.EmptyRootHash && len(body.Transactions) == 0 { + return nil, nil, fmt.Errorf("server returned empty transaction list but block header indicates transactions") + } + txs := make([]*types.Transaction, len(body.Transactions)) + for i, tx := range body.Transactions { + txs[i] = tx.tx + } + return types.NewBlockWithHeader(head).WithBody(txs, nil), body.UncleHashes, nil +} + // newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index. func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.Bytes { txs := b.Transactions() @@ -194,3 +290,31 @@ func sliceContainsHash(slice []string, hash common.Hash) int { } return 0 } + +func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { + arg := map[string]interface{}{ + "address": q.Addresses, + "topics": q.Topics, + } + if q.BlockHash != nil { + arg["blockHash"] = *q.BlockHash + if q.FromBlock != nil || q.ToBlock != nil { + return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + } + } else { + if q.FromBlock == nil { + arg["fromBlock"] = "0x0" + } else { + arg["fromBlock"] = toBlockNumArg(q.FromBlock) + } + arg["toBlock"] = toBlockNumArg(q.ToBlock) + } + return arg, nil +} + +func toBlockNumArg(number *big.Int) string { + if number == nil { + return "latest" + } + return hexutil.EncodeBig(number) +} diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 44a2f7e2..53edf503 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff" "github.com/vulcanize/ipld-eth-indexer/pkg/eth" "github.com/vulcanize/ipld-eth-indexer/pkg/ipfs" @@ -44,6 +45,65 @@ type RPCTransaction struct { S *hexutil.Big `json:"s"` } +// RPCReceipt represents a receipt that will serialize to the RPC representation of a receipt +type RPCReceipt struct { + BlockHash *common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + TransactionHash *common.Hash `json:"transactionHash"` + TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` + From common.Address `json:"from"` + To *common.Address `json:"to"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + CumulativeGsUsed hexutil.Uint64 `json:"cumulativeGasUsed"` + ContractAddress *common.Address `json:"contractAddress"` + Logs []*types.Log `json:"logs"` + Bloom types.Bloom `json:"logsBloom"` + Root []byte `json:"root"` + Status uint64 `json:"status"` +} + +// AccountResult struct for GetProof +type AccountResult struct { + Address common.Address `json:"address"` + AccountProof []string `json:"accountProof"` + Balance *hexutil.Big `json:"balance"` + CodeHash common.Hash `json:"codeHash"` + Nonce hexutil.Uint64 `json:"nonce"` + StorageHash common.Hash `json:"storageHash"` + StorageProof []StorageResult `json:"storageProof"` +} + +// StorageResult for GetProof +type StorageResult struct { + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` +} + +// CallArgs represents the arguments for a call. +type CallArgs struct { + From *common.Address `json:"from"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Value *hexutil.Big `json:"value"` + Data *hexutil.Bytes `json:"data"` +} + +// account indicates the overriding fields of account during the execution of +// a message call. +// Note, state and stateDiff can't be specified at the same time. If state is +// set, message execution will only use the data in the given state. Otherwise +// if statDiff is set, all diff will be applied first and then execute the call +// message. +type account struct { + Nonce *hexutil.Uint64 `json:"nonce"` + Code *hexutil.Bytes `json:"code"` + Balance **hexutil.Big `json:"balance"` + State *map[common.Hash]common.Hash `json:"state"` + StateDiff *map[common.Hash]common.Hash `json:"stateDiff"` +} + // IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server // Returned by IPLDFetcher and ResponseFilterer type IPLDs struct { From a480c28a67fbddcd716bc60adce6325bdabd3c85 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 28 Oct 2020 09:02:27 -0500 Subject: [PATCH 03/10] update readme --- README.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cca12140..2b98b518 100644 --- a/README.md +++ b/README.md @@ -93,13 +93,31 @@ TODO: Port the IPLD RPC subscription endpoints after the decoupling ipld-eth-server currently recapitulates portions of the Ethereum JSON-RPC api standard. The currently supported standard endpoints are: -`eth_call` +`eth_call` +`eth_getBalance` +`eth_getStorageAt` +`eth_getCode` +`eth_getProof` `eth_blockNumber` -`eth_getLogs` `eth_getHeaderByNumber` +`eth_getHeaderByHash` `eth_getBlockByNumber` `eth_getBlockByHash` +`eth_getTransactionCount` +`eth_getBlockTransactionCountByHash` +`eth_getBlockTransactionCountByNumber` `eth_getTransactionByHash` +`eth_getRawTransactionByHash` +`eth_getTransactionByBlockHashAndIndex` +`eth_getTransactionByBlockNumberAndIndex` +`eth_getRawTransactionByBlockHashAndIndex` +`eth_getRawTransactionByBlockNumberAndIndex` +`eth_getTransactionReceipt` +`eth_getLogs` +`eth_getUncleCountByBlockHash` +`eth_getUncleCountByBlockNumber` +`eth_getUncleByBlockHashAndIndex` +`eth_getUncleByBlockNumberAndIndex` TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice) From e1026d52615877a3218e2b09da1b0b52aae72c51 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 29 Oct 2020 14:59:09 -0500 Subject: [PATCH 04/10] remaining block endpoint unit test; uncle endpoints unit tests --- pkg/eth/api.go | 71 ++++---- pkg/eth/api_test.go | 258 ++++++++++++++++++++++++++---- pkg/eth/test_helpers/test_data.go | 49 ++++-- 3 files changed, 304 insertions(+), 74 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 81a5606c..f6e62018 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -94,7 +94,7 @@ func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.Block func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { header, err := pea.B.HeaderByHash(ctx, hash) if header != nil && err == nil { - if res, err := pea.rpcMarshalHeader(header); err != nil { + if res, err := pea.rpcMarshalHeader(header); err == nil { return res } } @@ -158,6 +158,12 @@ func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, f return nil, err } +/* + +Uncles + +*/ + // GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true // all transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { @@ -382,11 +388,21 @@ Receipts and Logs // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { + receipt, err := pea.localGetTransactionReceipt(ctx, hash) + if receipt != nil && err == nil { + return receipt, nil + } + if pea.rpc != nil { + if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil { + return receipt, nil + } + } + return nil, err +} + +func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) if err != nil { - if rct := pea.remoteGetTransactionReceipt(ctx, hash); rct != nil { - return rct, nil - } return nil, err } if tx == nil { @@ -394,9 +410,6 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common. } receipts, err := pea.B.GetReceipts(ctx, blockHash) if err != nil { - if rct := pea.remoteGetTransactionReceipt(ctx, hash); rct != nil { - return rct, nil - } return nil, err } if len(receipts) <= int(index) { @@ -441,24 +454,22 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common. } func (pea *PublicEthAPI) remoteGetTransactionReceipt(ctx context.Context, hash common.Hash) map[string]interface{} { - if pea.rpc != nil { - var rct *RPCReceipt - if err := pea.rpc.CallContext(ctx, &rct, "eth_getTransactionReceipt", hash); rct != nil && err == nil { - return map[string]interface{}{ - "blockHash": rct.BlockHash, - "blockNumber": rct.BlockNumber, - "transactionHash": rct.TransactionHash, - "transactionIndex": rct.TransactionIndex, - "from": rct.From, - "to": rct.To, - "gasUsed": rct.GasUsed, - "cumulativeGasUsed": rct.CumulativeGsUsed, - "contractAddress": rct.ContractAddress, - "logs": rct.Logs, - "logsBloom": rct.Bloom, - "root": rct.Root, - "status": rct.Status, - } + var rct *RPCReceipt + if err := pea.rpc.CallContext(ctx, &rct, "eth_getTransactionReceipt", hash); rct != nil && err == nil { + return map[string]interface{}{ + "blockHash": rct.BlockHash, + "blockNumber": rct.BlockNumber, + "transactionHash": rct.TransactionHash, + "transactionIndex": rct.TransactionIndex, + "from": rct.From, + "to": rct.To, + "gasUsed": rct.GasUsed, + "cumulativeGasUsed": rct.CumulativeGsUsed, + "contractAddress": rct.ContractAddress, + "logs": rct.Logs, + "logsBloom": rct.Bloom, + "root": rct.Root, + "status": rct.Status, } } return nil @@ -835,11 +846,13 @@ func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx boo if err != nil { return nil, err } - td, err := pea.B.GetTd(b.Hash()) - if err != nil { - return nil, err + if inclTx { + td, err := pea.B.GetTd(b.Hash()) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(td) } - fields["totalDifficulty"] = (*hexutil.Big)(td) return fields, err } diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index 8211ebf2..69066980 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -77,6 +77,46 @@ var ( "receiptsRoot": test_helpers.MockBlock.Header().ReceiptHash, "totalDifficulty": (*hexutil.Big)(test_helpers.MockBlock.Header().Difficulty), } + expectedUncle1 = map[string]interface{}{ + "number": (*hexutil.Big)(test_helpers.MockUncles[0].Number), + "hash": test_helpers.MockUncles[0].Hash(), + "parentHash": test_helpers.MockUncles[0].ParentHash, + "nonce": test_helpers.MockUncles[0].Nonce, + "mixHash": test_helpers.MockUncles[0].MixDigest, + "sha3Uncles": test_helpers.MockUncles[0].UncleHash, + "logsBloom": test_helpers.MockUncles[0].Bloom, + "stateRoot": test_helpers.MockUncles[0].Root, + "miner": test_helpers.MockUncles[0].Coinbase, + "difficulty": (*hexutil.Big)(test_helpers.MockUncles[0].Difficulty), + "extraData": hexutil.Bytes(test_helpers.MockUncles[0].Extra), + "size": hexutil.Uint64(types.NewBlockWithHeader(test_helpers.MockUncles[0]).Size()), + "gasLimit": hexutil.Uint64(test_helpers.MockUncles[0].GasLimit), + "gasUsed": hexutil.Uint64(test_helpers.MockUncles[0].GasUsed), + "timestamp": hexutil.Uint64(test_helpers.MockUncles[0].Time), + "transactionsRoot": test_helpers.MockUncles[0].TxHash, + "receiptsRoot": test_helpers.MockUncles[0].ReceiptHash, + "uncles": []common.Hash{}, + } + expectedUncle2 = map[string]interface{}{ + "number": (*hexutil.Big)(test_helpers.MockUncles[1].Number), + "hash": test_helpers.MockUncles[1].Hash(), + "parentHash": test_helpers.MockUncles[1].ParentHash, + "nonce": test_helpers.MockUncles[1].Nonce, + "mixHash": test_helpers.MockUncles[1].MixDigest, + "sha3Uncles": test_helpers.MockUncles[1].UncleHash, + "logsBloom": test_helpers.MockUncles[1].Bloom, + "stateRoot": test_helpers.MockUncles[1].Root, + "miner": test_helpers.MockUncles[1].Coinbase, + "difficulty": (*hexutil.Big)(test_helpers.MockUncles[1].Difficulty), + "extraData": hexutil.Bytes(test_helpers.MockUncles[1].Extra), + "size": hexutil.Uint64(types.NewBlockWithHeader(test_helpers.MockUncles[1]).Size()), + "gasLimit": hexutil.Uint64(test_helpers.MockUncles[1].GasLimit), + "gasUsed": hexutil.Uint64(test_helpers.MockUncles[1].GasUsed), + "timestamp": hexutil.Uint64(test_helpers.MockUncles[1].Time), + "transactionsRoot": test_helpers.MockUncles[1].TxHash, + "receiptsRoot": test_helpers.MockUncles[1].ReceiptHash, + "uncles": []common.Hash{}, + } expectedTransaction = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0) ) @@ -107,6 +147,40 @@ var _ = Describe("API", func() { AfterEach(func() { eth.TearDownDB(db) }) + /* + + Headers and blocks + + */ + Describe("GetHeaderByHash", func() { + It("Retrieves a header by hash", func() { + hash := test_helpers.MockBlock.Header().Hash() + header := api.GetHeaderByHash(context.Background(), hash) + Expect(header).To(Equal(expectedHeader)) + }) + }) + + Describe("GetHeaderByNumber", func() { + It("Retrieves a header by number", func() { + number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) + Expect(err).ToNot(HaveOccurred()) + header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(header).To(Equal(expectedHeader)) + }) + + It("Throws an error if a header cannot be found", func() { + number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) + Expect(err).ToNot(HaveOccurred()) + header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number+1)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + Expect(header).To(BeNil()) + _, err = api.B.DB.Beginx() + Expect(err).ToNot(HaveOccurred()) + }) + }) + Describe("BlockNumber", func() { It("Retrieves the head block number", func() { bn := api.BlockNumber() @@ -116,15 +190,6 @@ var _ = Describe("API", func() { }) }) - Describe("GetTransactionByHash", func() { - It("Retrieves a transaction by hash", func() { - hash := test_helpers.MockTransactions[0].Hash() - tx, err := api.GetTransactionByHash(context.Background(), hash) - Expect(err).ToNot(HaveOccurred()) - Expect(tx).To(Equal(expectedTransaction)) - }) - }) - Describe("GetBlockByNumber", func() { It("Retrieves a block by number", func() { // without full txs @@ -154,27 +219,6 @@ var _ = Describe("API", func() { }) }) - Describe("GetHeaderByNumber", func() { - It("Retrieves a header by number", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number)) - Expect(err).ToNot(HaveOccurred()) - Expect(header).To(Equal(expectedHeader)) - }) - - It("Throws an error if a header cannot be found", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number+1)) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) - Expect(header).To(BeNil()) - _, err = api.B.DB.Beginx() - Expect(err).ToNot(HaveOccurred()) - }) - }) - Describe("GetBlockByHash", func() { It("Retrieves a block by hash", func() { // without full txs @@ -202,6 +246,129 @@ var _ = Describe("API", func() { }) }) + /* + + Uncles + + */ + + Describe("GetUncleByBlockNumberAndIndex", func() { + It("Retrieves the uncle at the provided index in the canoncial block with the provided hash", func() { + number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) + Expect(err).ToNot(HaveOccurred()) + uncle1, err := api.GetUncleByBlockNumberAndIndex(context.Background(), rpc.BlockNumber(number), 0) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle1).To(Equal(expectedUncle1)) + uncle2, err := api.GetUncleByBlockNumberAndIndex(context.Background(), rpc.BlockNumber(number), 1) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle2).To(Equal(expectedUncle2)) + }) + }) + + Describe("GetUncleByBlockHashAndIndex", func() { + It("Retrieves the uncle at the provided index in the block with the provided hash", func() { + hash := test_helpers.MockBlock.Header().Hash() + uncle1, err := api.GetUncleByBlockHashAndIndex(context.Background(), hash, 0) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle1).To(Equal(expectedUncle1)) + uncle2, err := api.GetUncleByBlockHashAndIndex(context.Background(), hash, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle2).To(Equal(expectedUncle2)) + }) + }) + + Describe("GetUncleCountByBlockNumber", func() { + It("Retrieves the number of uncles for the canonical block with the provided number", func() { + number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) + Expect(err).ToNot(HaveOccurred()) + count := api.GetUncleCountByBlockNumber(context.Background(), rpc.BlockNumber(number)) + Expect(uint64(*count)).To(Equal(uint64(2))) + }) + }) + + Describe("GetUncleCountByBlockHash", func() { + It("Retrieves the number of uncles for the block with the provided hash", func() { + hash := test_helpers.MockBlock.Header().Hash() + count := api.GetUncleCountByBlockHash(context.Background(), hash) + Expect(uint64(*count)).To(Equal(uint64(2))) + }) + }) + + /* + + Transactions + + */ + + Describe("GetTransactionCount", func() { + It("Retrieves the number of transactions the given address has sent for the given block number or block hash", func() { + + }) + }) + + Describe("GetBlockTransactionCountByNumber", func() { + It("Retrieves the number of transactions in the canonical block with the provided number", func() { + + }) + }) + + Describe("GetBlockTransactionCountByHash", func() { + It("Retrieves the number of transactions in the block with the provided hash ", func() { + + }) + }) + + Describe("GetTransactionByBlockNumberAndIndex", func() { + It("Retrieves the tx with the provided index in the canonical block with the provided block number", func() { + + }) + }) + + Describe("GetTransactionByBlockHashAndIndex", func() { + It("Retrieves the tx with the provided index in the block with the provided hash", func() { + + }) + }) + + Describe("GetRawTransactionByBlockNumberAndIndex", func() { + It("Retrieves the raw tx with the provided index in the canonical block with the provided block number", func() { + + }) + }) + + Describe("GetRawTransactionByBlockHashAndIndex", func() { + It("Retrieves the raw tx with the provided index in the block with the provided hash", func() { + + }) + }) + + Describe("GetTransactionByHash", func() { + It("Retrieves a transaction by hash", func() { + hash := test_helpers.MockTransactions[0].Hash() + tx, err := api.GetTransactionByHash(context.Background(), hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(expectedTransaction)) + }) + }) + + Describe("GetRawTransactionByHash", func() { + It("Retrieves a raw transaction by hash", func() { + + }) + }) + + /* + + Receipts and logs + + */ + + Describe("GetTransactionReceipt", func() { + It("Retrieves a receipt by tx hash", func() { + + }) + }) + Describe("GetLogs", func() { It("Retrieves receipt logs that match the provided topics within the provided range", func() { crit := ethereum.FilterQuery{ @@ -572,4 +739,35 @@ var _ = Describe("API", func() { Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) }) }) + + /* + + State and storage + + */ + + Describe("GetBalance", func() { + It("Retrieves the eth balance for the provided account address at the block with the provided hash or number", func() { + + }) + }) + + Describe("GetStorageAt", func() { + It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { + + }) + }) + + Describe("GetCode", func() { + It("Retrieves the code for the provided contract address at the block with the provied hash or number", func() { + + }) + }) + + Describe("GetProof", func() { + It("Retrieves the Merkle-proof for a given account and optionally some storage keys at the block with the provided hash or number", func() { + + }) + }) + }) diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index 08f57c34..b90e43fd 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -57,19 +57,39 @@ var ( Extra: []byte{}, } MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts() - ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) - MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) - MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) - Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") - AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") - ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) - ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String() - MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} - mockTopic11 = common.HexToHash("0x04") - mockTopic12 = common.HexToHash("0x06") - mockTopic21 = common.HexToHash("0x05") - mockTopic22 = common.HexToHash("0x07") - MockLog1 = &types.Log{ + MockUncles = []*types.Header{ + { + Time: 1, + Number: new(big.Int).Add(BlockNumber, big.NewInt(1)), + Root: common.HexToHash("0x1"), + TxHash: common.HexToHash("0x1"), + ReceiptHash: common.HexToHash("0x1"), + Difficulty: big.NewInt(500001), + Extra: []byte{}, + }, + { + Time: 2, + Number: new(big.Int).Add(BlockNumber, big.NewInt(2)), + Root: common.HexToHash("0x2"), + TxHash: common.HexToHash("0x2"), + ReceiptHash: common.HexToHash("0x2"), + Difficulty: big.NewInt(500002), + Extra: []byte{}, + }, + } + ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) + MockBlock = types.NewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String() + MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} + mockTopic11 = common.HexToHash("0x04") + mockTopic12 = common.HexToHash("0x06") + mockTopic21 = common.HexToHash("0x05") + mockTopic22 = common.HexToHash("0x07") + MockLog1 = &types.Log{ Address: Address, Topics: []common.Hash{mockTopic11, mockTopic12}, Data: []byte{}, @@ -266,7 +286,6 @@ var ( nonce0 = uint64(0) AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") - accountPath = common.Bytes2Hex([]byte{'\x0c'}) AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e") AccountLeafKey = testhelpers.Account2LeafKey Account, _ = rlp.EncodeToBytes(state.Account{ @@ -341,7 +360,7 @@ var ( CID: HeaderCID.String(), MhKey: HeaderMhKey, TotalDifficulty: MockBlock.Difficulty().String(), - Reward: "5000000000000000000", + Reward: "5312500000000000000", StateRoot: MockBlock.Root().String(), RctRoot: MockBlock.ReceiptHash().String(), TxRoot: MockBlock.TxHash().String(), From dc25ea7f8730bf01ff9de3437b775c2d9ab1bcdf Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 29 Oct 2020 15:18:27 -0500 Subject: [PATCH 05/10] new transaction endpoint unit tests (except for GetTransactionCount which is still TODO because testing its current implementation requires complete state (e.g. like eth_call test) so- ideally- will first refactor it to use state diff 2ndary indexes rather than proceeding through regular ethdb interface) --- pkg/eth/api_test.go | 170 ++++++++++++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 54 deletions(-) diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index 69066980..ba632773 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -20,6 +20,8 @@ import ( "context" "strconv" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -37,6 +39,9 @@ import ( ) var ( + number = rpc.BlockNumber(test_helpers.BlockNumber.Int64()) + blockHash = test_helpers.MockBlock.Header().Hash() + ctx = context.Background() expectedBlock = map[string]interface{}{ "number": (*hexutil.Big)(test_helpers.MockBlock.Number()), "hash": test_helpers.MockBlock.Hash(), @@ -59,7 +64,7 @@ var ( } expectedHeader = map[string]interface{}{ "number": (*hexutil.Big)(test_helpers.MockBlock.Header().Number), - "hash": test_helpers.MockBlock.Header().Hash(), + "hash": blockHash, "parentHash": test_helpers.MockBlock.Header().ParentHash, "nonce": test_helpers.MockBlock.Header().Nonce, "mixHash": test_helpers.MockBlock.Header().MixDigest, @@ -117,7 +122,12 @@ var ( "receiptsRoot": test_helpers.MockUncles[1].ReceiptHash, "uncles": []common.Hash{}, } - expectedTransaction = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0) + expectedTransaction = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0) + expectedTransaction2 = eth.NewRPCTransaction(test_helpers.MockTransactions[1], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 1) + expectedTransaction3 = eth.NewRPCTransaction(test_helpers.MockTransactions[2], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 2) + expectRawTx, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[0]) + expectRawTx2, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[1]) + expectRawTx3, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[2]) ) var _ = Describe("API", func() { @@ -154,25 +164,20 @@ var _ = Describe("API", func() { */ Describe("GetHeaderByHash", func() { It("Retrieves a header by hash", func() { - hash := test_helpers.MockBlock.Header().Hash() - header := api.GetHeaderByHash(context.Background(), hash) + header := api.GetHeaderByHash(ctx, blockHash) Expect(header).To(Equal(expectedHeader)) }) }) Describe("GetHeaderByNumber", func() { It("Retrieves a header by number", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number)) + header, err := api.GetHeaderByNumber(ctx, number) Expect(err).ToNot(HaveOccurred()) Expect(header).To(Equal(expectedHeader)) }) It("Throws an error if a header cannot be found", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number+1)) + header, err := api.GetHeaderByNumber(ctx, rpc.BlockNumber(number+1)) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) Expect(header).To(BeNil()) @@ -193,9 +198,7 @@ var _ = Describe("API", func() { Describe("GetBlockByNumber", func() { It("Retrieves a block by number", func() { // without full txs - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - block, err := api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), false) + block, err := api.GetBlockByNumber(ctx, number, false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -206,7 +209,7 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } // with full txs - block, err = api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), true) + block, err = api.GetBlockByNumber(ctx, number, true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -222,7 +225,7 @@ var _ = Describe("API", func() { Describe("GetBlockByHash", func() { It("Retrieves a block by hash", func() { // without full txs - block, err := api.GetBlockByHash(context.Background(), test_helpers.MockBlock.Hash(), false) + block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -233,7 +236,7 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } // with full txs - block, err = api.GetBlockByHash(context.Background(), test_helpers.MockBlock.Hash(), true) + block, err = api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -254,12 +257,10 @@ var _ = Describe("API", func() { Describe("GetUncleByBlockNumberAndIndex", func() { It("Retrieves the uncle at the provided index in the canoncial block with the provided hash", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - uncle1, err := api.GetUncleByBlockNumberAndIndex(context.Background(), rpc.BlockNumber(number), 0) + uncle1, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 0) Expect(err).ToNot(HaveOccurred()) Expect(uncle1).To(Equal(expectedUncle1)) - uncle2, err := api.GetUncleByBlockNumberAndIndex(context.Background(), rpc.BlockNumber(number), 1) + uncle2, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 1) Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) @@ -267,11 +268,10 @@ var _ = Describe("API", func() { Describe("GetUncleByBlockHashAndIndex", func() { It("Retrieves the uncle at the provided index in the block with the provided hash", func() { - hash := test_helpers.MockBlock.Header().Hash() - uncle1, err := api.GetUncleByBlockHashAndIndex(context.Background(), hash, 0) + uncle1, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 0) Expect(err).ToNot(HaveOccurred()) Expect(uncle1).To(Equal(expectedUncle1)) - uncle2, err := api.GetUncleByBlockHashAndIndex(context.Background(), hash, 1) + uncle2, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 1) Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) @@ -279,17 +279,14 @@ var _ = Describe("API", func() { Describe("GetUncleCountByBlockNumber", func() { It("Retrieves the number of uncles for the canonical block with the provided number", func() { - number, err := strconv.ParseInt(test_helpers.BlockNumber.String(), 10, 64) - Expect(err).ToNot(HaveOccurred()) - count := api.GetUncleCountByBlockNumber(context.Background(), rpc.BlockNumber(number)) + count := api.GetUncleCountByBlockNumber(ctx, number) Expect(uint64(*count)).To(Equal(uint64(2))) }) }) Describe("GetUncleCountByBlockHash", func() { It("Retrieves the number of uncles for the block with the provided hash", func() { - hash := test_helpers.MockBlock.Header().Hash() - count := api.GetUncleCountByBlockHash(context.Background(), hash) + count := api.GetUncleCountByBlockHash(ctx, blockHash) Expect(uint64(*count)).To(Equal(uint64(2))) }) }) @@ -308,52 +305,117 @@ var _ = Describe("API", func() { Describe("GetBlockTransactionCountByNumber", func() { It("Retrieves the number of transactions in the canonical block with the provided number", func() { - + count := api.GetBlockTransactionCountByNumber(ctx, number) + Expect(uint64(*count)).To(Equal(uint64(3))) }) }) Describe("GetBlockTransactionCountByHash", func() { It("Retrieves the number of transactions in the block with the provided hash ", func() { - + count := api.GetBlockTransactionCountByHash(ctx, blockHash) + Expect(uint64(*count)).To(Equal(uint64(3))) }) }) Describe("GetTransactionByBlockNumberAndIndex", func() { It("Retrieves the tx with the provided index in the canonical block with the provided block number", func() { + tx := api.GetTransactionByBlockNumberAndIndex(ctx, number, 0) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction)) + tx = api.GetTransactionByBlockNumberAndIndex(ctx, number, 1) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction2)) + + tx = api.GetTransactionByBlockNumberAndIndex(ctx, number, 2) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction3)) }) }) Describe("GetTransactionByBlockHashAndIndex", func() { It("Retrieves the tx with the provided index in the block with the provided hash", func() { + tx := api.GetTransactionByBlockHashAndIndex(ctx, blockHash, 0) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction)) + tx = api.GetTransactionByBlockHashAndIndex(ctx, blockHash, 1) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction2)) + + tx = api.GetTransactionByBlockHashAndIndex(ctx, blockHash, 2) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(expectedTransaction3)) }) }) Describe("GetRawTransactionByBlockNumberAndIndex", func() { It("Retrieves the raw tx with the provided index in the canonical block with the provided block number", func() { + tx := api.GetRawTransactionByBlockNumberAndIndex(ctx, number, 0) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx))) + tx = api.GetRawTransactionByBlockNumberAndIndex(ctx, number, 1) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx2))) + + tx = api.GetRawTransactionByBlockNumberAndIndex(ctx, number, 2) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx3))) }) }) Describe("GetRawTransactionByBlockHashAndIndex", func() { It("Retrieves the raw tx with the provided index in the block with the provided hash", func() { + tx := api.GetRawTransactionByBlockHashAndIndex(ctx, blockHash, 0) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx))) + tx = api.GetRawTransactionByBlockHashAndIndex(ctx, blockHash, 1) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx2))) + + tx = api.GetRawTransactionByBlockHashAndIndex(ctx, blockHash, 2) + Expect(tx).ToNot(BeNil()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx3))) }) }) Describe("GetTransactionByHash", func() { It("Retrieves a transaction by hash", func() { hash := test_helpers.MockTransactions[0].Hash() - tx, err := api.GetTransactionByHash(context.Background(), hash) + tx, err := api.GetTransactionByHash(ctx, hash) Expect(err).ToNot(HaveOccurred()) Expect(tx).To(Equal(expectedTransaction)) + + hash = test_helpers.MockTransactions[1].Hash() + tx, err = api.GetTransactionByHash(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(expectedTransaction2)) + + hash = test_helpers.MockTransactions[2].Hash() + tx, err = api.GetTransactionByHash(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(expectedTransaction3)) }) }) Describe("GetRawTransactionByHash", func() { It("Retrieves a raw transaction by hash", func() { + hash := test_helpers.MockTransactions[0].Hash() + tx, err := api.GetRawTransactionByHash(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx))) + hash = test_helpers.MockTransactions[1].Hash() + tx, err = api.GetRawTransactionByHash(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx2))) + + hash = test_helpers.MockTransactions[2].Hash() + tx, err = api.GetRawTransactionByHash(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(tx).To(Equal(hexutil.Bytes(expectRawTx3))) }) }) @@ -380,7 +442,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err := api.GetLogs(context.Background(), crit) + logs, err := api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -395,7 +457,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -410,7 +472,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -427,7 +489,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(0)) @@ -443,7 +505,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -460,7 +522,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -478,7 +540,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -497,7 +559,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -512,7 +574,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -527,7 +589,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -537,7 +599,7 @@ var _ = Describe("API", func() { FromBlock: test_helpers.MockBlock.Number(), ToBlock: test_helpers.MockBlock.Number(), } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -554,7 +616,7 @@ var _ = Describe("API", func() { }, }, } - logs, err := api.GetLogs(context.Background(), crit) + logs, err := api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -570,7 +632,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -584,7 +646,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -600,7 +662,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -616,7 +678,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(0)) @@ -632,7 +694,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2})) @@ -646,7 +708,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -664,7 +726,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -673,7 +735,7 @@ var _ = Describe("API", func() { BlockHash: &hash, Topics: [][]common.Hash{}, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -697,7 +759,7 @@ var _ = Describe("API", func() { }, }, } - logs, err := api.GetLogs(context.Background(), crit) + logs, err := api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(1)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1})) @@ -720,7 +782,7 @@ var _ = Describe("API", func() { }, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) @@ -733,7 +795,7 @@ var _ = Describe("API", func() { test_helpers.AnotherAddress, }, } - logs, err = api.GetLogs(context.Background(), crit) + logs, err = api.GetLogs(ctx, crit) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2})) From cffceb53dbae2c1ca0987a5557ec33e4df7d581e Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 29 Oct 2020 22:07:39 -0500 Subject: [PATCH 06/10] optimize GetTransactionCount, GetBalance, and GetCode to use secondary indexes instead of operating through ethdb where we have to iterate down trie from root to leaf (multiple db lookups) to access account info --- pkg/eth/api.go | 52 ++++++++++++---------- pkg/eth/backend.go | 93 ++++++++++++++++++++++++++++++++++++++- pkg/eth/ipld_retriever.go | 22 ++++----- 3 files changed, 132 insertions(+), 35 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index f6e62018..3ad6401a 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -244,14 +244,9 @@ Transactions // GetTransactionCount returns the number of transactions the given address has sent for the given block number func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { - // Resolve block number and use its state to ask for the nonce - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - nonce := state.GetNonce(address) - err = state.Error() - if err == nil { - return (*hexutil.Uint64)(&nonce), nil - } + count, err := pea.localGetTransactionCount(ctx, address, blockNrOrHash) + if count != nil && err == nil { + return count, nil } if pea.rpc != nil { var num *hexutil.Uint64 @@ -262,6 +257,15 @@ func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common return nil, err } +func (pea *PublicEthAPI) localGetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { + account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash) + if err != nil { + return nil, err + } + nonce := hexutil.Uint64(account.Nonce) + return &nonce, nil +} + // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil { @@ -401,6 +405,7 @@ func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common. } func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { + // TODO: this can be optimized for Postgres tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash) if err != nil { return nil, err @@ -444,7 +449,7 @@ func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash co fields["status"] = hexutil.Uint(receipt.Status) } if receipt.Logs == nil { - fields["logs"] = [][]*types.Log{} + fields["logs"] = []*types.Log{} } // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation if receipt.ContractAddress != (common.Address{}) { @@ -492,6 +497,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery) } func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) { + // TODO: this can be optimized away from using the old cid retriever and ipld fetcher interfaces // Convert FilterQuery into ReceiptFilter addrStrs := make([]string, len(crit.Addresses)) for i, addr := range crit.Addresses { @@ -588,13 +594,9 @@ State and Storage // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - balance := state.GetBalance(address) - err = state.Error() - if err == nil { - return (*hexutil.Big)(balance), nil - } + bal, err := pea.localGetBalance(ctx, address, blockNrOrHash) + if bal != nil && err == nil { + return bal, nil } if pea.rpc != nil { var res *hexutil.Big @@ -605,6 +607,14 @@ func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, return nil, err } +func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { + account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash) + if err != nil { + return nil, err + } + return (*hexutil.Big)(account.Balance), nil +} + // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. @@ -628,13 +638,9 @@ func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Addres // GetCode returns the code stored at the given address in the state for the given block number. func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - code := state.GetCode(address) - err = state.Error() - if err == nil { - return code, nil - } + code, err := pea.B.GetCodeByNumberOrHash(ctx, address, blockNrOrHash) + if code != nil && err == nil { + return code, nil } if pea.rpc != nil { var res hexutil.Bytes diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 5911a569..f711b0cf 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -19,6 +19,7 @@ package eth import ( "context" "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -30,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -38,6 +40,8 @@ import ( ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres" + shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared" + "github.com/vulcanize/ipld-eth-server/pkg/shared" ) @@ -59,6 +63,12 @@ const ( WHERE blocks.key = transaction_cids.mh_key AND transaction_cids.header_id = header_cids.id AND transaction_cids.tx_hash = $1` + RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids + WHERE state_accounts.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND state_leaf_key = $1 + AND block_hash = $2` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) type Backend struct { @@ -417,7 +427,7 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece if err != nil { return nil, err } - rcts := make(types.Receipts, 0, len(receiptBytes)) + rcts := make(types.Receipts, len(receiptBytes)) for i, rctBytes := range receiptBytes { rct := new(types.Receipt) if err := rlp.DecodeBytes(rctBytes, rct); err != nil { @@ -512,6 +522,87 @@ func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.Sta return vm.NewEVM(c, state, b.Config.ChainConfig, b.Config.VmConfig), nil } +// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash +func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*state.Account, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetAccountByNumber(ctx, address, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetAccountByHash(ctx, address, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height +func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, number uint64) (*state.Account, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetAccountByHash(ctx, address, hash) +} + +// GetAccountByHash returns the account object for the provided address at the block with the provided hash +func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*state.Account, error) { + _, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash) + if err != nil { + return nil, err + } + acct := new(state.Account) + return acct, rlp.DecodeBytes(accountRlp, acct) +} + +// GetCodeByNumberOrHash returns the byte code for the contract deployed at the provided address at the block with the provided hash or block number +func (b *Backend) GetCodeByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) ([]byte, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetCodeByNumber(ctx, address, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetCodeByHash(ctx, address, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetCodeByNumber returns the byte code for the contract deployed at the provided address at the canonical block with the provided block number +func (b *Backend) GetCodeByNumber(ctx context.Context, address common.Address, number uint64) ([]byte, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetCodeByHash(ctx, address, hash) +} + +// GetCodeByHash returns the byte code for the contract deployed at the provided address at the block with the provided hash +func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, hash common.Hash) ([]byte, error) { + codeHash := make([]byte, 0) + leafKey := crypto.Keccak256Hash(address.Bytes()) + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + if err := tx.Get(&codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex()); err != nil { + return nil, err + } + mhKey, err := shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash)) + if err != nil { + return nil, err + } + code := make([]byte, 0) + err = tx.Get(&code, RetrieveCodeByMhKey, mhKey) + return code, err +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index b2800300..4d00712f 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -40,11 +40,11 @@ const ( RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveUnclesByBlockHashPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveUnclesByBlockNumberPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_number = $1` @@ -54,41 +54,41 @@ const ( RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveTransactionsByBlockHashPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveTransactionsByBlockNumberPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_number = $1` RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveReceiptsByTxHashesPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks + RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveReceiptsByBlockHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveReceiptsByBlockNumberPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_number = $1` - RetrieveReceiptByTxHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids + RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 AND block_hash = $2` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 @@ -333,7 +333,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr } // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number -// This can return multiple results if we have two versions of state in the database as the provided height +// This can return multiple results if we have two versions of state in the database at the provided height func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) { accountResults := make([]ipldResult, 0) leafKey := crypto.Keccak256Hash(address.Bytes()) From b664aee6218f2f9dba57c94b1e85910e847307da Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 29 Oct 2020 22:08:26 -0500 Subject: [PATCH 07/10] unit tests for GetTransactionCount, GetTransactionReceipt, GetBalance, and GetCode --- pkg/eth/api_test.go | 169 ++++++++++++++++++++++++++---- pkg/eth/backend.go | 7 +- pkg/eth/eth_call_test.go | 19 +++- pkg/eth/ipld_retriever.go | 11 +- pkg/eth/ipld_retriever_test.go | 156 --------------------------- pkg/eth/test_helpers/test_data.go | 7 +- 6 files changed, 186 insertions(+), 183 deletions(-) delete mode 100644 pkg/eth/ipld_retriever_test.go diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index ba632773..f7d052dd 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -20,18 +20,19 @@ import ( "context" "strconv" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth" "github.com/vulcanize/ipld-eth-indexer/pkg/postgres" + shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared" "github.com/vulcanize/ipld-eth-server/pkg/eth" "github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers" @@ -39,6 +40,8 @@ import ( ) var ( + randomAddr = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f") + randomHash = crypto.Keccak256Hash(randomAddr.Bytes()) number = rpc.BlockNumber(test_helpers.BlockNumber.Int64()) blockHash = test_helpers.MockBlock.Header().Hash() ctx = context.Background() @@ -128,6 +131,48 @@ var ( expectRawTx, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[0]) expectRawTx2, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[1]) expectRawTx3, _ = rlp.EncodeToBytes(test_helpers.MockTransactions[2]) + expectedReceipt = map[string]interface{}{ + "blockHash": blockHash, + "blockNumber": hexutil.Uint64(uint64(number.Int64())), + "transactionHash": expectedTransaction.Hash, + "transactionIndex": hexutil.Uint64(0), + "from": expectedTransaction.From, + "to": expectedTransaction.To, + "gasUsed": hexutil.Uint64(test_helpers.MockReceipts[0].GasUsed), + "cumulativeGasUsed": hexutil.Uint64(test_helpers.MockReceipts[0].CumulativeGasUsed), + "contractAddress": nil, + "logs": test_helpers.MockReceipts[0].Logs, + "logsBloom": test_helpers.MockReceipts[0].Bloom, + "root": hexutil.Bytes(test_helpers.MockReceipts[0].PostState), + } + expectedReceipt2 = map[string]interface{}{ + "blockHash": blockHash, + "blockNumber": hexutil.Uint64(uint64(number.Int64())), + "transactionHash": expectedTransaction2.Hash, + "transactionIndex": hexutil.Uint64(1), + "from": expectedTransaction2.From, + "to": expectedTransaction2.To, + "gasUsed": hexutil.Uint64(test_helpers.MockReceipts[1].GasUsed), + "cumulativeGasUsed": hexutil.Uint64(test_helpers.MockReceipts[1].CumulativeGasUsed), + "contractAddress": nil, + "logs": test_helpers.MockReceipts[1].Logs, + "logsBloom": test_helpers.MockReceipts[1].Bloom, + "root": hexutil.Bytes(test_helpers.MockReceipts[1].PostState), + } + expectedReceipt3 = map[string]interface{}{ + "blockHash": blockHash, + "blockNumber": hexutil.Uint64(uint64(number.Int64())), + "transactionHash": expectedTransaction3.Hash, + "transactionIndex": hexutil.Uint64(2), + "from": expectedTransaction3.From, + "to": expectedTransaction3.To, + "gasUsed": hexutil.Uint64(test_helpers.MockReceipts[2].GasUsed), + "cumulativeGasUsed": hexutil.Uint64(test_helpers.MockReceipts[2].CumulativeGasUsed), + "contractAddress": nil, + "logs": test_helpers.MockReceipts[2].Logs, + "logsBloom": test_helpers.MockReceipts[2].Bloom, + "root": hexutil.Bytes(test_helpers.MockReceipts[2].PostState), + } ) var _ = Describe("API", func() { @@ -137,7 +182,8 @@ var _ = Describe("API", func() { backend *eth.Backend api *eth.PublicEthAPI ) - BeforeEach(func() { + // Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database + It("", func() { var err error db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) @@ -147,6 +193,8 @@ var _ = Describe("API", func() { api = eth.NewPublicEthAPI(backend, nil) err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) + err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode) + Expect(err).ToNot(HaveOccurred()) uncles := test_helpers.MockBlock.Uncles() uncleHashes := make([]common.Hash, len(uncles)) for i, uncle := range uncles { @@ -154,21 +202,13 @@ var _ = Describe("API", func() { } expectedBlock["uncles"] = uncleHashes }) - AfterEach(func() { - eth.TearDownDB(db) - }) + // Single test db tear down at end of all tests + defer It("", func() { eth.TearDownDB(db) }) /* Headers and blocks */ - Describe("GetHeaderByHash", func() { - It("Retrieves a header by hash", func() { - header := api.GetHeaderByHash(ctx, blockHash) - Expect(header).To(Equal(expectedHeader)) - }) - }) - Describe("GetHeaderByNumber", func() { It("Retrieves a header by number", func() { header, err := api.GetHeaderByNumber(ctx, number) @@ -298,8 +338,23 @@ var _ = Describe("API", func() { */ Describe("GetTransactionCount", func() { - It("Retrieves the number of transactions the given address has sent for the given block number or block hash", func() { + It("Retrieves the number of transactions the given address has sent for the given block number", func() { + count, err := api.GetTransactionCount(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(*count).To(Equal(hexutil.Uint64(1))) + count, err = api.GetTransactionCount(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(*count).To(Equal(hexutil.Uint64(0))) + }) + It("Retrieves the number of transactions the given address has sent for the given block hash", func() { + count, err := api.GetTransactionCount(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).ToNot(HaveOccurred()) + Expect(*count).To(Equal(hexutil.Uint64(1))) + + count, err = api.GetTransactionCount(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).ToNot(HaveOccurred()) + Expect(*count).To(Equal(hexutil.Uint64(0))) }) }) @@ -398,6 +453,10 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(tx).To(Equal(expectedTransaction3)) }) + It("Throws an error if it cannot find a tx for the provided tx hash", func() { + _, err := api.GetTransactionByHash(ctx, randomHash) + Expect(err).To(HaveOccurred()) + }) }) Describe("GetRawTransactionByHash", func() { @@ -417,6 +476,10 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(tx).To(Equal(hexutil.Bytes(expectRawTx3))) }) + It("Throws an error if it cannot find a tx for the provided tx hash", func() { + _, err := api.GetRawTransactionByHash(ctx, randomHash) + Expect(err).To(HaveOccurred()) + }) }) /* @@ -427,7 +490,24 @@ var _ = Describe("API", func() { Describe("GetTransactionReceipt", func() { It("Retrieves a receipt by tx hash", func() { + hash := test_helpers.MockTransactions[0].Hash() + rct, err := api.GetTransactionReceipt(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(rct).To(Equal(expectedReceipt)) + hash = test_helpers.MockTransactions[1].Hash() + rct, err = api.GetTransactionReceipt(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(rct).To(Equal(expectedReceipt2)) + + hash = test_helpers.MockTransactions[2].Hash() + rct, err = api.GetTransactionReceipt(ctx, hash) + Expect(err).ToNot(HaveOccurred()) + Expect(rct).To(Equal(expectedReceipt3)) + }) + It("Throws an error if it cannot find a receipt for the provided tx hash", func() { + _, err := api.GetTransactionReceipt(ctx, randomHash) + Expect(err).To(HaveOccurred()) }) }) @@ -809,21 +889,56 @@ var _ = Describe("API", func() { */ Describe("GetBalance", func() { - It("Retrieves the eth balance for the provided account address at the block with the provided hash or number", func() { + It("Retrieves the eth balance for the provided account address at the block with the provided number", func() { + bal, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal((*hexutil.Big)(test_helpers.AccountBalance))) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal((*hexutil.Big)(common.Big0))) + }) + It("Retrieves the eth balance for the provided account address at the block with the provided hash", func() { + bal, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal((*hexutil.Big)(test_helpers.AccountBalance))) + + bal, err = api.GetBalance(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal((*hexutil.Big)(common.Big0))) + }) + It("Throws an error for an account it cannot find the balance for", func() { + _, err := api.GetBalance(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).To(HaveOccurred()) }) }) Describe("GetStorageAt", func() { It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { - + /* + val, err := api.GetStorageAt(ctx, test_helpers.ContractAddress, common.Bytes2Hex(test_helpers.StorageLeafKey), rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal((hexutil.Bytes)(test_helpers.StorageValue))) + */ }) }) Describe("GetCode", func() { - It("Retrieves the code for the provided contract address at the block with the provied hash or number", func() { - + It("Retrieves the code for the provided contract address at the block with the provided number", func() { + code, err := api.GetCode(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) }) + It("Retrieves the code for the provided contract address at the block with the provided hash", func() { + code, err := api.GetCode(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) + }) + It("Throws an error for an account it cannot find the code for", func() { + _, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true)) + Expect(err).To(HaveOccurred()) + }) + }) Describe("GetProof", func() { @@ -831,5 +946,21 @@ var _ = Describe("API", func() { }) }) - }) + +func publishCode(db *postgres.DB, codeHash common.Hash, code []byte) error { + tx, err := db.Beginx() + if err != nil { + return err + } + mhKey, err := shared2.MultihashKeyFromKeccak256(codeHash) + if err != nil { + tx.Rollback() + return err + } + if err := shared2.PublishDirect(tx, mhKey, code); err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index f711b0cf..dc68cd32 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -67,7 +67,12 @@ const ( WHERE state_accounts.state_id = state_cids.id AND state_cids.header_id = header_cids.id AND state_leaf_key = $1 - AND block_hash = $2` + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $2) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) diff --git a/pkg/eth/eth_call_test.go b/pkg/eth/eth_call_test.go index 0ad599df..c5360d16 100644 --- a/pkg/eth/eth_call_test.go +++ b/pkg/eth/eth_call_test.go @@ -89,7 +89,7 @@ var _ = Describe("eth_call", func() { api = eth.NewPublicEthAPI(backend, nil) // make the test blockchain (and state) - blocks, receipts, chain = test_helpers.MakeChain(4, test_helpers.Genesis, test_helpers.TestChainGen) + blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen) pams = statediff.Params{ IntermediateStateNodes: true, IntermediateStorageNodes: true, @@ -146,7 +146,17 @@ var _ = Describe("eth_call", func() { To: &test_helpers.ContractAddr, Data: &bdata, } - res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil) + // Before contract deployment + res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(BeNil()) + + res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(BeNil()) + + // After deployment + res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil) Expect(err).ToNot(HaveOccurred()) expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) Expect(res).To(Equal(expectedRes)) @@ -160,6 +170,11 @@ var _ = Describe("eth_call", func() { Expect(err).ToNot(HaveOccurred()) expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009")) Expect(res).To(Equal(expectedRes)) + + res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil) + Expect(err).ToNot(HaveOccurred()) + expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")) + Expect(res).To(Equal(expectedRes)) }) }) }) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 4d00712f..a5c2a1aa 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -87,12 +87,19 @@ const ( WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 - AND block_hash = $2` + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $2) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 - AND block_number = $2` + AND block_number <= $2 + ORDER BY block_number DESC + LIMIT 1` ) type ipldResult struct { diff --git a/pkg/eth/ipld_retriever_test.go b/pkg/eth/ipld_retriever_test.go deleted file mode 100644 index 544b16d6..00000000 --- a/pkg/eth/ipld_retriever_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package eth_test - -/* -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth" - "github.com/vulcanize/ipld-eth-indexer/pkg/postgres" - - "github.com/vulcanize/ipld-eth-server/pkg/eth" - "github.com/vulcanize/ipld-eth-server/pkg/eth/mocks" - "github.com/vulcanize/ipld-eth-server/pkg/shared" -) - -var _ = Describe("IPLD Retriever", func() { - var ( - db *postgres.DB - repo *eth2.IPLDPublisher - //retriever *eth.IPLDRetriever - ) - BeforeEach(func() { - var err error - db, err = shared.SetupDB() - Expect(err).ToNot(HaveOccurred()) - repo = eth2.NewIPLDPublisher(db) - //retriever = eth.NewIPLDRetriever(db) - err = repo.Publish(mocks.MockConvertedPayload) - Expect(err).ToNot(HaveOccurred()) - err = repo.Publish(mocks.MockConvertedPayload2) - Expect(err).ToNot(HaveOccurred()) - }) - AfterEach(func() { - eth.TearDownDB(db) - }) - - Describe("RetrieveHeadersByHashes", func() { - It("Retrieves all of the headers that correspond to the provided hashes", func() { - - }) - }) - - Describe("RetrieveHeadersByBlockNumber", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveHeaderByHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveUnclesByHashes", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveUnclesByBlockHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveUnclesByBlockNumber", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveUncleByHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveTransactionsByHashes", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveTransactionsByBlockHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveTransactionsByBlockNumber", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveTransactionByTxHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveReceiptsByTxHashes", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveReceiptsByBlockHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveReceiptsByBlockNumber", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveReceiptByHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveAccountByAddressAndBlockHash", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - - Describe("RetrieveAccountByAddressAndBlockNumber", func() { - It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { - - }) - }) - -}) -*/ diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index b90e43fd..c86854d7 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -259,7 +259,7 @@ var ( // statediff data storageLocation = common.HexToHash("0") StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() - StorageValue = common.Hex2Bytes("01") + StorageValue = crypto.Keccak256([]byte{1, 2, 3, 4, 5}) StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ StoragePartialPath, @@ -268,7 +268,7 @@ var ( nonce1 = uint64(1) ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" - ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") + ContractCodeHash = crypto.Keccak256Hash(MockContractByteCode) contractPath = common.Bytes2Hex([]byte{'\x06'}) ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress) ContractAccount, _ = rlp.EncodeToBytes(state.Account{ @@ -284,13 +284,14 @@ var ( }) nonce0 = uint64(0) + AccountBalance = big.NewInt(1000) AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e") AccountLeafKey = testhelpers.Account2LeafKey Account, _ = rlp.EncodeToBytes(state.Account{ Nonce: nonce0, - Balance: big.NewInt(1000), + Balance: AccountBalance, CodeHash: AccountCodeHash.Bytes(), Root: common.HexToHash(AccountRoot), }) From b208281ad6f9c00d350b4d3b73ada1894a4b73f3 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 30 Oct 2020 11:54:22 -0500 Subject: [PATCH 08/10] optimize GetStorageAt; GetStorageAt unit tests --- pkg/eth/api.go | 10 +- pkg/eth/api_test.go | 135 ++++++++----- pkg/eth/backend.go | 28 +++ .../{eth_call_test.go => eth_state_test.go} | 66 ++++-- pkg/eth/ipld_retriever.go | 191 +++++++++++++++--- pkg/eth/test_helpers/chain_maker.go | 20 +- 6 files changed, 334 insertions(+), 116 deletions(-) rename pkg/eth/{eth_call_test.go => eth_state_test.go} (67%) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 3ad6401a..8be45588 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -619,13 +619,9 @@ func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Add // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { - state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state != nil && err == nil { - res := state.GetState(address, common.HexToHash(key)) - err = state.Error() - if err == nil { - return res[:], nil - } + storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash) + if storageVal != nil && err == nil { + return storageVal, nil } if pea.rpc != nil { var res hexutil.Bytes diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index f7d052dd..f910c7e5 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -43,6 +43,7 @@ var ( randomAddr = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f") randomHash = crypto.Keccak256Hash(randomAddr.Bytes()) number = rpc.BlockNumber(test_helpers.BlockNumber.Int64()) + wrongNumber = rpc.BlockNumber(number + 1) blockHash = test_helpers.MockBlock.Header().Hash() ctx = context.Background() expectedBlock = map[string]interface{}{ @@ -177,18 +178,17 @@ var ( var _ = Describe("API", func() { var ( - db *postgres.DB - indexAndPublisher *eth2.IPLDPublisher - backend *eth.Backend - api *eth.PublicEthAPI + db *postgres.DB + api *eth.PublicEthAPI ) // Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database - It("", func() { + // Note: if you focus one of the tests be sure to focus this and the defered It() + It("test init", func() { var err error db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) - indexAndPublisher = eth2.NewIPLDPublisher(db) - backend, err = eth.NewEthBackend(db, ð.Config{}) + indexAndPublisher := eth2.NewIPLDPublisher(db) + backend, err := eth.NewEthBackend(db, ð.Config{}) Expect(err).ToNot(HaveOccurred()) api = eth.NewPublicEthAPI(backend, nil) err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload) @@ -203,13 +203,13 @@ var _ = Describe("API", func() { expectedBlock["uncles"] = uncleHashes }) // Single test db tear down at end of all tests - defer It("", func() { eth.TearDownDB(db) }) + defer It("test teardown", func() { eth.TearDownDB(db) }) /* Headers and blocks */ - Describe("GetHeaderByNumber", func() { + Describe("eth_getHeaderByNumber", func() { It("Retrieves a header by number", func() { header, err := api.GetHeaderByNumber(ctx, number) Expect(err).ToNot(HaveOccurred()) @@ -217,7 +217,7 @@ var _ = Describe("API", func() { }) It("Throws an error if a header cannot be found", func() { - header, err := api.GetHeaderByNumber(ctx, rpc.BlockNumber(number+1)) + header, err := api.GetHeaderByNumber(ctx, wrongNumber) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) Expect(header).To(BeNil()) @@ -226,7 +226,19 @@ var _ = Describe("API", func() { }) }) - Describe("BlockNumber", func() { + Describe("eth_getHeaderByHash", func() { + It("Retrieves a header by hash", func() { + header := api.GetHeaderByHash(ctx, blockHash) + Expect(header).To(Equal(expectedHeader)) + }) + + It("Throws an error if a header cannot be found", func() { + header := api.GetHeaderByHash(ctx, randomHash) + Expect(header).To(BeNil()) + }) + }) + + Describe("eth_blockNumber", func() { It("Retrieves the head block number", func() { bn := api.BlockNumber() ubn := (uint64)(bn) @@ -235,9 +247,8 @@ var _ = Describe("API", func() { }) }) - Describe("GetBlockByNumber", func() { - It("Retrieves a block by number", func() { - // without full txs + Describe("eth_getBlockByNumber", func() { + It("Retrieves a block by number, without full txs", func() { block, err := api.GetBlockByNumber(ctx, number, false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) @@ -248,8 +259,9 @@ var _ = Describe("API", func() { for key, val := range expectedBlock { Expect(val).To(Equal(block[key])) } - // with full txs - block, err = api.GetBlockByNumber(ctx, number, true) + }) + It("Retrieves a block by number, with full txs", func() { + block, err := api.GetBlockByNumber(ctx, number, true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -260,11 +272,15 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } }) + It("Throws an error if a block cannot be found", func() { + _, err := api.GetBlockByNumber(ctx, wrongNumber, false) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) }) - Describe("GetBlockByHash", func() { - It("Retrieves a block by hash", func() { - // without full txs + Describe("eth_getBlockByHash", func() { + It("Retrieves a block by hash, without full txs", func() { block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), false) Expect(err).ToNot(HaveOccurred()) transactionHashes := make([]interface{}, len(test_helpers.MockBlock.Transactions())) @@ -275,8 +291,9 @@ var _ = Describe("API", func() { for key, val := range expectedBlock { Expect(val).To(Equal(block[key])) } - // with full txs - block, err = api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) + }) + It("Retrieves a block by hash, with full txs", func() { + block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true) Expect(err).ToNot(HaveOccurred()) transactions := make([]interface{}, len(test_helpers.MockBlock.Transactions())) for i, trx := range test_helpers.MockBlock.Transactions() { @@ -287,6 +304,11 @@ var _ = Describe("API", func() { Expect(val).To(Equal(block[key])) } }) + It("Throws an error if a block cannot be found", func() { + _, err := api.GetBlockByHash(ctx, randomHash, false) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) }) /* @@ -295,7 +317,7 @@ var _ = Describe("API", func() { */ - Describe("GetUncleByBlockNumberAndIndex", func() { + Describe("eth_getUncleByBlockNumberAndIndex", func() { It("Retrieves the uncle at the provided index in the canoncial block with the provided hash", func() { uncle1, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 0) Expect(err).ToNot(HaveOccurred()) @@ -304,9 +326,19 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) + It("Throws an error if an block for blocknumber cannot be found", func() { + _, err := api.GetUncleByBlockNumberAndIndex(ctx, wrongNumber, 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Returns `nil` if an uncle at the provided index does not exist for the block found for the provided block number", func() { + uncle, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle).To(BeNil()) + }) }) - Describe("GetUncleByBlockHashAndIndex", func() { + Describe("eth_getUncleByBlockHashAndIndex", func() { It("Retrieves the uncle at the provided index in the block with the provided hash", func() { uncle1, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 0) Expect(err).ToNot(HaveOccurred()) @@ -315,16 +347,26 @@ var _ = Describe("API", func() { Expect(err).ToNot(HaveOccurred()) Expect(uncle2).To(Equal(expectedUncle2)) }) + It("Throws an error if an block for blockhash cannot be found", func() { + _, err := api.GetUncleByBlockHashAndIndex(ctx, randomHash, 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Returns `nil` if an uncle at the provided index does not exist for the block with the provided hash", func() { + uncle, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(uncle).To(BeNil()) + }) }) - Describe("GetUncleCountByBlockNumber", func() { + Describe("eth_getUncleCountByBlockNumber", func() { It("Retrieves the number of uncles for the canonical block with the provided number", func() { count := api.GetUncleCountByBlockNumber(ctx, number) Expect(uint64(*count)).To(Equal(uint64(2))) }) }) - Describe("GetUncleCountByBlockHash", func() { + Describe("eth_getUncleCountByBlockHash", func() { It("Retrieves the number of uncles for the block with the provided hash", func() { count := api.GetUncleCountByBlockHash(ctx, blockHash) Expect(uint64(*count)).To(Equal(uint64(2))) @@ -337,7 +379,7 @@ var _ = Describe("API", func() { */ - Describe("GetTransactionCount", func() { + Describe("eth_getTransactionCount", func() { It("Retrieves the number of transactions the given address has sent for the given block number", func() { count, err := api.GetTransactionCount(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -358,21 +400,21 @@ var _ = Describe("API", func() { }) }) - Describe("GetBlockTransactionCountByNumber", func() { + Describe("eth_getBlockTransactionCountByNumber", func() { It("Retrieves the number of transactions in the canonical block with the provided number", func() { count := api.GetBlockTransactionCountByNumber(ctx, number) Expect(uint64(*count)).To(Equal(uint64(3))) }) }) - Describe("GetBlockTransactionCountByHash", func() { + Describe("eth_getBlockTransactionCountByHash", func() { It("Retrieves the number of transactions in the block with the provided hash ", func() { count := api.GetBlockTransactionCountByHash(ctx, blockHash) Expect(uint64(*count)).To(Equal(uint64(3))) }) }) - Describe("GetTransactionByBlockNumberAndIndex", func() { + Describe("eth_getTransactionByBlockNumberAndIndex", func() { It("Retrieves the tx with the provided index in the canonical block with the provided block number", func() { tx := api.GetTransactionByBlockNumberAndIndex(ctx, number, 0) Expect(tx).ToNot(BeNil()) @@ -388,7 +430,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetTransactionByBlockHashAndIndex", func() { + Describe("eth_getTransactionByBlockHashAndIndex", func() { It("Retrieves the tx with the provided index in the block with the provided hash", func() { tx := api.GetTransactionByBlockHashAndIndex(ctx, blockHash, 0) Expect(tx).ToNot(BeNil()) @@ -404,7 +446,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByBlockNumberAndIndex", func() { + Describe("eth_getRawTransactionByBlockNumberAndIndex", func() { It("Retrieves the raw tx with the provided index in the canonical block with the provided block number", func() { tx := api.GetRawTransactionByBlockNumberAndIndex(ctx, number, 0) Expect(tx).ToNot(BeNil()) @@ -420,7 +462,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByBlockHashAndIndex", func() { + Describe("eth_getRawTransactionByBlockHashAndIndex", func() { It("Retrieves the raw tx with the provided index in the block with the provided hash", func() { tx := api.GetRawTransactionByBlockHashAndIndex(ctx, blockHash, 0) Expect(tx).ToNot(BeNil()) @@ -436,7 +478,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetTransactionByHash", func() { + Describe("eth_getTransactionByHash", func() { It("Retrieves a transaction by hash", func() { hash := test_helpers.MockTransactions[0].Hash() tx, err := api.GetTransactionByHash(ctx, hash) @@ -459,7 +501,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetRawTransactionByHash", func() { + Describe("eth_getRawTransactionByHash", func() { It("Retrieves a raw transaction by hash", func() { hash := test_helpers.MockTransactions[0].Hash() tx, err := api.GetRawTransactionByHash(ctx, hash) @@ -488,7 +530,7 @@ var _ = Describe("API", func() { */ - Describe("GetTransactionReceipt", func() { + Describe("eth_getTransactionReceipt", func() { It("Retrieves a receipt by tx hash", func() { hash := test_helpers.MockTransactions[0].Hash() rct, err := api.GetTransactionReceipt(ctx, hash) @@ -511,7 +553,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetLogs", func() { + Describe("eth_getLogs", func() { It("Retrieves receipt logs that match the provided topics within the provided range", func() { crit := ethereum.FilterQuery{ Topics: [][]common.Hash{ @@ -888,7 +930,7 @@ var _ = Describe("API", func() { */ - Describe("GetBalance", func() { + Describe("eth_getBalance", func() { It("Retrieves the eth balance for the provided account address at the block with the provided number", func() { bal, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -913,17 +955,7 @@ var _ = Describe("API", func() { }) }) - Describe("GetStorageAt", func() { - It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { - /* - val, err := api.GetStorageAt(ctx, test_helpers.ContractAddress, common.Bytes2Hex(test_helpers.StorageLeafKey), rpc.BlockNumberOrHashWithNumber(number)) - Expect(err).ToNot(HaveOccurred()) - Expect(val).To(Equal((hexutil.Bytes)(test_helpers.StorageValue))) - */ - }) - }) - - Describe("GetCode", func() { + Describe("eth_getCode", func() { It("Retrieves the code for the provided contract address at the block with the provided number", func() { code, err := api.GetCode(ctx, test_helpers.ContractAddress, rpc.BlockNumberOrHashWithNumber(number)) Expect(err).ToNot(HaveOccurred()) @@ -938,13 +970,6 @@ var _ = Describe("API", func() { _, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true)) Expect(err).To(HaveOccurred()) }) - - }) - - Describe("GetProof", func() { - It("Retrieves the Merkle-proof for a given account and optionally some storage keys at the block with the provided hash or number", func() { - - }) }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index dc68cd32..0cb4c52c 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -22,6 +22,8 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" @@ -608,6 +610,32 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has return code, err } +// GetStorageByNumberOrHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided number or hash +func (b *Backend) GetStorageByNumberOrHash(ctx context.Context, address common.Address, storageLeafKey common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.GetStorageByNumber(ctx, address, storageLeafKey, uint64(blockNr.Int64())) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return b.GetStorageByHash(ctx, address, storageLeafKey, hash) + } + return nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// GetStorageByNumber returns the storage value for the provided contract address an storage key at the block corresponding to the provided number +func (b *Backend) GetStorageByNumber(ctx context.Context, address common.Address, storageLeafKey common.Hash, number uint64) (hexutil.Bytes, error) { + hash := b.GetCanonicalHash(number) + if hash == (common.Hash{}) { + return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number) + } + return b.GetStorageByHash(ctx, address, storageLeafKey, hash) +} + +// GetStorageByHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided hash +func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, storageLeafKey, hash common.Hash) (hexutil.Bytes, error) { + _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address, storageLeafKey, hash) + return storageRlp, err +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/eth_call_test.go b/pkg/eth/eth_state_test.go similarity index 67% rename from pkg/eth/eth_call_test.go rename to pkg/eth/eth_state_test.go index c5360d16..6709764d 100644 --- a/pkg/eth/eth_call_test.go +++ b/pkg/eth/eth_state_test.go @@ -59,28 +59,23 @@ func init() { } } -var _ = Describe("eth_call", func() { +var _ = Describe("eth state reading tests", func() { var ( blocks []*types.Block receipts []types.Receipts chain *core.BlockChain db *postgres.DB - transformer *eth2.StateDiffTransformer - backend *eth.Backend api *eth.PublicEthAPI - builder statediff.Builder - pams statediff.Params chainConfig = params.TestChainConfig mockTD = big.NewInt(1337) ) - - BeforeEach(func() { + It("test init", func() { // db and type initializations var err error db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) - transformer = eth2.NewStateDiffTransformer(chainConfig, db) - backend, err = eth.NewEthBackend(db, ð.Config{ + transformer := eth2.NewStateDiffTransformer(chainConfig, db) + backend, err := eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, VmConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), @@ -90,12 +85,12 @@ var _ = Describe("eth_call", func() { // make the test blockchain (and state) blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen) - pams = statediff.Params{ + params := statediff.Params{ IntermediateStateNodes: true, IntermediateStorageNodes: true, } // iterate over the blocks, generating statediff payloads, and transforming the data into Postgres - builder = statediff.NewBuilder(chain.StateCache()) + builder := statediff.NewBuilder(chain.StateCache()) for i, block := range blocks { var args statediff.Args var rcts types.Receipts @@ -115,7 +110,7 @@ var _ = Describe("eth_call", func() { } rcts = receipts[i-1] } - diff, err := builder.BuildStateDiffObject(args, pams) + diff, err := builder.BuildStateDiffObject(args, params) Expect(err).ToNot(HaveOccurred()) diffRlp, err := rlp.EncodeToBytes(diff) Expect(err).ToNot(HaveOccurred()) @@ -133,10 +128,11 @@ var _ = Describe("eth_call", func() { Expect(err).ToNot(HaveOccurred()) } }) - AfterEach(func() { + defer It("test teardown", func() { eth.TearDownDB(db) chain.Stop() }) + Describe("eth_call", func() { It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() { data, err := parsedABI.Pack("data") @@ -146,7 +142,7 @@ var _ = Describe("eth_call", func() { To: &test_helpers.ContractAddr, Data: &bdata, } - // Before contract deployment + // Before contract deployment, returns nil res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil) Expect(err).ToNot(HaveOccurred()) Expect(res).To(BeNil()) @@ -177,4 +173,46 @@ var _ = Describe("eth_call", func() { Expect(res).To(Equal(expectedRes)) }) }) + + Describe("eth_getStorageAt", func() { + It("Throws an error if it tries to access a contract which does not exist", func() { + _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + + _, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Throws an error if it tries to access a contract slot which does not exist", func() { + _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() { + // After deployment + val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + expectedRes := hexutil.Bytes(common.Hex2Bytes("01")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + expectedRes = hexutil.Bytes(common.Hex2Bytes("03")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + expectedRes = hexutil.Bytes(common.Hex2Bytes("09")) + Expect(val).To(Equal(expectedRes)) + + val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal(hexutil.Bytes{})) + }) + }) + + Describe("eth_getProof", func() { + + }) }) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index a5c2a1aa..057fd6f9 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,6 +19,8 @@ package eth import ( "fmt" + "github.com/vulcanize/ipld-eth-server/pkg/shared" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -28,62 +30,78 @@ import ( ) const ( - RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids + RetrieveIPLDpgStr = `SELECT data + FROM public.blocks + WHERE key = $1` + RetrieveHeadersByHashesPgStr = `SELECT cid, data + FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_number = $1` - RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids + RetrieveHeaderByHashPgStr = `SELECT cid, data + FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = $1` - RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids + RetrieveUnclesByHashesPgStr = `SELECT cid, data + FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks + RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids, eth.header_cids, public.blocks WHERE uncle_cids.header_id = header_cids.id AND uncle_cids.mh_key = blocks.key AND block_number = $1` - RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids + RetrieveUncleByHashPgStr = `SELECT cid, data + FROM eth.uncle_cids INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = $1` - RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids + RetrieveTransactionsByHashesPgStr = `SELECT cid, data + FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks + RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids, eth.header_cids, public.blocks WHERE transaction_cids.header_id = header_cids.id AND transaction_cids.mh_key = blocks.key AND block_number = $1` RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks + RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, public.blocks WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_hash = $1` - RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks + RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id AND receipt_cids.mh_key = blocks.key AND block_number = $1` - RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids + RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data + FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids WHERE receipt_cids.mh_key = blocks.key AND receipt_cids.tx_id = transaction_cids.id AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data + FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 @@ -93,13 +111,59 @@ const ( AND header_cids.id = (SELECT canonical_header(block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids, eth.header_cids, public.blocks + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data + FROM eth.state_cids, eth.header_cids, public.blocks WHERE state_cids.header_id = header_cids.id AND state_cids.mh_key = blocks.key AND state_leaf_key = $1 AND block_number <= $2 ORDER BY block_number DESC LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= $3 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` + retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` + wasNodeDeletedpgStr = `SELECT exists(SELECT * + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE storage_cids.state_id = state_cids.id + AND storage_path = $1 + AND block_number > $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND storage_cids.node_type = 3)` ) type ipldResult struct { @@ -323,6 +387,7 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, } // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash +// TODO: ensure this handles deleted accounts appropriately func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { accountResult := new(ipldResult) leafKey := crypto.Keccak256Hash(address.Bytes()) @@ -340,25 +405,87 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr } // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number -// This can return multiple results if we have two versions of state in the database at the provided height -func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) { - accountResults := make([]ipldResult, 0) +// This can return a non-canonical account +func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) { + accountResult := new(ipldResult) leafKey := crypto.Keccak256Hash(address.Bytes()) - if err := r.db.Get(&accountResults, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { - return nil, nil, err + if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { + return "", nil, err } - cids := make([]string, len(accountResults)) - accounts := make([][]byte, len(accountResults)) - for i, res := range accountResults { - cids[i] = res.CID - var iface []interface{} - if err := rlp.DecodeBytes(res.Data, &iface); err != nil { - return nil, nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) - } - if len(iface) != 2 { - return nil, nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") - } - accounts[i] = iface[1].([]byte) + var i []interface{} + if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) } - return cids, accounts, nil + if len(i) != 2 { + return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") + } + return accountResult.CID, i[1].([]byte), nil +} + +type storageInfo struct { + CID string `db:"cid"` + Data []byte `db:"data"` + Path []byte `db:"storage_path"` + BlockNumber uint64 `db:"block_number"` +} + +// RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash +func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) { + // Begin tx + tx, err := r.db.Beginx() + if err != nil { + return "", nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + storageResult := new(storageInfo) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + if err := tx.Get(storageResult, retrieveStorageInfoPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { + return "", nil, err + } + + deleted := false + if err := tx.Get(&deleted, wasNodeDeletedpgStr, storageResult.Path, storageResult.BlockNumber, hash.Hex()); err != nil { + return "", nil, err + } + if deleted { + return "", []byte{}, nil + } + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + return "", nil, err + } + if len(i) != 2 { + err = fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") + return "", nil, err + } + return storageResult.CID, i[1].([]byte), err +} + +// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number +// This can retrun a non-canonical value +func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) { + storageResult := new(ipldResult) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { + return "", nil, err + } + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") + } + return storageResult.CID, i[1].([]byte), nil } diff --git a/pkg/eth/test_helpers/chain_maker.go b/pkg/eth/test_helpers/chain_maker.go index 2e587a9d..e694ce67 100644 --- a/pkg/eth/test_helpers/chain_maker.go +++ b/pkg/eth/test_helpers/chain_maker.go @@ -37,12 +37,16 @@ var ( TestBankFunds = big.NewInt(100000000) Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) - Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") - ContractAddr common.Address + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") + ContractAddr common.Address + IndexZero = "0000000000000000000000000000000000000000000000000000000000000000" + IndexOne = "0000000000000000000000000000000000000000000000000000000000000001" + ContractSlotPosition = common.FromHex(IndexOne) + ContractSlotKeyHash = crypto.Keccak256Hash(ContractSlotPosition) ) /* test function signatures @@ -88,8 +92,8 @@ func TestChainGen(i int, block *core.BlockGen) { case 3: block.SetCoinbase(Account2Addr) data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009") - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) - block.AddTx(tx1) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) case 4: block.SetCoinbase(Account1Addr) data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000") From 7c06d4b3a1c95edbdfcac1b03934743890d4caf8 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 30 Oct 2020 12:14:26 -0500 Subject: [PATCH 09/10] additional unit tests --- pkg/eth/eth_state_test.go | 218 +++++++++++++++++++++++++++- pkg/eth/ipld_retriever.go | 40 ++--- pkg/eth/test_helpers/chain_maker.go | 7 +- 3 files changed, 230 insertions(+), 35 deletions(-) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 6709764d..58148816 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -66,6 +66,7 @@ var _ = Describe("eth state reading tests", func() { chain *core.BlockChain db *postgres.DB api *eth.PublicEthAPI + backend *eth.Backend chainConfig = params.TestChainConfig mockTD = big.NewInt(1337) ) @@ -75,7 +76,7 @@ var _ = Describe("eth state reading tests", func() { db, err = shared.SetupDB() Expect(err).ToNot(HaveOccurred()) transformer := eth2.NewStateDiffTransformer(chainConfig, db) - backend, err := eth.NewEthBackend(db, ð.Config{ + backend, err = eth.NewEthBackend(db, ð.Config{ ChainConfig: chainConfig, VmConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), @@ -174,6 +175,217 @@ var _ = Describe("eth state reading tests", func() { }) }) + var ( + expectedContractBalance = (*hexutil.Big)(common.Big0) + expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds) + + expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000)) + expectedBankBalanceBlock1 = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000))) + + expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000)) + expectedBankBalanceBlock2 = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000))) + + expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward)) + + expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward)) + + expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward)) + ) + + Describe("eth_getBalance", func() { + It("Retrieves the eth balance for the provided account address at the block with the provided number", func() { + bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock0)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + _, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock1)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock2)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock3)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock4)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock5)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock4)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + }) + It("Retrieves the eth balance for the provided account address at the block with the provided hash", func() { + bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock0)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + _, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock1)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock2)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock3)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock1)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock4)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + + bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct1BalanceBlock5)) + bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedAcct2BalanceBlock4)) + bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedContractBalance)) + bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(bal).To(Equal(expectedBankBalanceBlock2)) + }) + It("Throws an error for an account it cannot find the balance for an account at the provided block number", func() { + _, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + It("Throws an error for an account it cannot find the balance for an account at the provided block hash", func() { + _, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + _, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sql: no rows in result set")) + }) + }) + + Describe("eth_getCode", func() { + It("Retrieves the code for the provided contract address at the block with the provided number", func() { + code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) + + code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) + }) + It("Retrieves the code for the provided contract address at the block with the provided hash", func() { + code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) + + code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true)) + Expect(err).ToNot(HaveOccurred()) + Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode))) + }) + It("Throws an error for an account it cannot find the code for", func() { + _, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true)) + Expect(err).To(HaveOccurred()) + }) + It("Throws an error for a contract that doesn't exist at this hieght", func() { + _, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0)) + Expect(err).To(HaveOccurred()) + }) + }) + Describe("eth_getStorageAt", func() { It("Throws an error if it tries to access a contract which does not exist", func() { _, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0)) @@ -211,8 +423,4 @@ var _ = Describe("eth state reading tests", func() { Expect(val).To(Equal(hexutil.Bytes{})) }) }) - - Describe("eth_getProof", func() { - - }) }) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 057fd6f9..487058cd 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -30,9 +30,6 @@ import ( ) const ( - RetrieveIPLDpgStr = `SELECT data - FROM public.blocks - WHERE key = $1` RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) @@ -129,32 +126,19 @@ const ( AND block_number <= $3 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND header_cids.id = (SELECT canonical_header(block_number)) - ORDER BY block_number DESC - LIMIT 1` retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND header_cids.id = (SELECT canonical_header(block_number)) - ORDER BY block_number DESC - LIMIT 1` + FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND storage_cids.mh_key = blocks.key + AND state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` wasNodeDeletedpgStr = `SELECT exists(SELECT * FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE storage_cids.state_id = state_cids.id diff --git a/pkg/eth/test_helpers/chain_maker.go b/pkg/eth/test_helpers/chain_maker.go index e694ce67..1444b0d9 100644 --- a/pkg/eth/test_helpers/chain_maker.go +++ b/pkg/eth/test_helpers/chain_maker.go @@ -41,12 +41,15 @@ var ( Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") + DeploymentTxData = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032") + CodeHash = crypto.Keccak256Hash(ContractCode) ContractAddr common.Address IndexZero = "0000000000000000000000000000000000000000000000000000000000000000" IndexOne = "0000000000000000000000000000000000000000000000000000000000000001" ContractSlotPosition = common.FromHex(IndexOne) ContractSlotKeyHash = crypto.Keccak256Hash(ContractSlotPosition) + MiningReward = big.NewInt(2000000000000000000) ) /* test function signatures @@ -79,7 +82,7 @@ func TestChainGen(i int, block *core.BlockGen) { nonce := block.TxNonce(Account1Addr) tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key) nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key) + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key) ContractAddr = crypto.CreateAddress(Account1Addr, nonce) block.AddTx(tx1) block.AddTx(tx2) From b128f894c450e3be19dcf6a3d655d1fcf0e03bdb Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sat, 31 Oct 2020 15:00:03 -0500 Subject: [PATCH 10/10] postgres functions to check if node was removed (e.g. a la EIP158) in a range; update backend to use these to retrieve state and storage leafs in single (albeit complex) SELECT query --- ...0018_create_check_if_removed_functions.sql | 37 +++ db/schema.sql | 39 +++ pkg/eth/ipld_retriever.go | 246 +++++++++--------- 3 files changed, 195 insertions(+), 127 deletions(-) create mode 100644 db/migrations/00018_create_check_if_removed_functions.sql diff --git a/db/migrations/00018_create_check_if_removed_functions.sql b/db/migrations/00018_create_check_if_removed_functions.sql new file mode 100644 index 00000000..9a7e42d3 --- /dev/null +++ b/db/migrations/00018_create_check_if_removed_functions.sql @@ -0,0 +1,37 @@ +-- +goose Up +-- +goose StatementBegin +-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash +CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN +AS $$ + SELECT exists(SELECT * + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE storage_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND storage_cids.node_type = 3); +$$ LANGUAGE SQL; +-- +goose StatementEnd + +-- +goose StatementBegin +-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash +CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN +AS $$ + SELECT exists(SELECT * + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND state_cids.node_type = 3); +$$ LANGUAGE SQL; +-- +goose StatementEnd + +-- +goose Down +DROP FUNCTION was_storage_removed; +DROP FUNCTION was_state_removed; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 02e92f20..91be03f0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -100,6 +100,45 @@ CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint $$; +-- +-- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT exists(SELECT * + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND state_cids.node_type = 3); +$$; + + +-- +-- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT exists(SELECT * + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE storage_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND storage_cids.node_type = 3); +$$; + + SET default_tablespace = ''; SET default_table_access_method = heap; diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 487058cd..8418bfe6 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,8 +19,6 @@ package eth import ( "fmt" - "github.com/vulcanize/ipld-eth-server/pkg/shared" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -32,122 +30,130 @@ import ( const ( RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data + FROM eth.header_cids + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_number = $1` RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = $1` RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks - WHERE uncle_cids.header_id = header_cids.id - AND uncle_cids.mh_key = blocks.key - AND block_hash = $1` + RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data - FROM eth.uncle_cids, eth.header_cids, public.blocks - WHERE uncle_cids.header_id = header_cids.id - AND uncle_cids.mh_key = blocks.key - AND block_number = $1` + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + WHERE block_number = $1` RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = $1` RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids, eth.header_cids, public.blocks - WHERE transaction_cids.header_id = header_cids.id - AND transaction_cids.mh_key = blocks.key - AND block_hash = $1` + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids, eth.header_cids, public.blocks - WHERE transaction_cids.header_id = header_cids.id - AND transaction_cids.mh_key = blocks.key - AND block_number = $1` - RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + WHERE block_number = $1` + RetrieveTransactionByHashPgStr = `SELECT cid, data + FROM eth.transaction_cids + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, public.blocks - WHERE receipt_cids.mh_key = blocks.key - AND receipt_cids.tx_id = transaction_cids.id - AND tx_hash = ANY($1::VARCHAR(66)[])` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id - AND receipt_cids.mh_key = blocks.key - AND block_hash = $1` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id - AND receipt_cids.mh_key = blocks.key - AND block_number = $1` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE block_number = $1` RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids - WHERE receipt_cids.mh_key = blocks.key - AND receipt_cids.tx_id = transaction_cids.id - AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data - FROM eth.state_cids, eth.header_cids, public.blocks - WHERE state_cids.header_id = header_cids.id - AND state_cids.mh_key = blocks.key - AND state_leaf_key = $1 + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE tx_hash = $1` + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, + data, + was_state_removed(state_path, block_number, $2) AS removed + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) AND header_cids.id = (SELECT canonical_header(block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data - FROM eth.state_cids, eth.header_cids, public.blocks - WHERE state_cids.header_id = header_cids.id - AND state_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND block_number <= $2 - ORDER BY block_number DESC - LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= $3 - ORDER BY block_number DESC - LIMIT 1` - retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND header_cids.id = (SELECT canonical_header(block_number)) - ORDER BY block_number DESC - LIMIT 1` - wasNodeDeletedpgStr = `SELECT exists(SELECT * - FROM eth.storage_cids, eth.state_cids, eth.header_cids - WHERE storage_cids.state_id = state_cids.id - AND storage_path = $1 - AND block_number > $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND storage_cids.node_type = 3)` + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, + data, + was_state_removed(state_path, block_number, (SELECT block_hash + FROM eth.header_cids + WHERE block_number = $2 + LIMIT 1)) AS removed + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND block_number <= $2 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, + data, + was_storage_removed(storage_path, block_number, (SELECT block_hash + FROM eth.header_cids + WHERE block_number = $3 + LIMIT 1)) AS removed + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= $3 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, + data, + was_storage_removed(storage_path, block_number, $3) AS removed + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` ) type ipldResult struct { @@ -370,14 +376,23 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) } +type nodeInfo struct { + CID string `db:"cid"` + Data []byte `db:"data"` + Removed bool `db:"removed"` +} + // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash // TODO: ensure this handles deleted accounts appropriately func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { - accountResult := new(ipldResult) + accountResult := new(nodeInfo) leafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } + if accountResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -391,11 +406,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number // This can return a non-canonical account func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) { - accountResult := new(ipldResult) + accountResult := new(nodeInfo) leafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { return "", nil, err } + if accountResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -406,42 +424,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad return accountResult.CID, i[1].([]byte), nil } -type storageInfo struct { - CID string `db:"cid"` - Data []byte `db:"data"` - Path []byte `db:"storage_path"` - BlockNumber uint64 `db:"block_number"` -} - // RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) { - // Begin tx - tx, err := r.db.Beginx() - if err != nil { - return "", nil, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - storageResult := new(storageInfo) + storageResult := new(nodeInfo) stateLeafKey := crypto.Keccak256Hash(address.Bytes()) - if err := tx.Get(storageResult, retrieveStorageInfoPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } - - deleted := false - if err := tx.Get(&deleted, wasNodeDeletedpgStr, storageResult.Path, storageResult.BlockNumber, hash.Hex()); err != nil { - return "", nil, err - } - if deleted { + if storageResult.Removed { return "", []byte{}, nil } var i []interface{} @@ -450,20 +440,22 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(addr return "", nil, err } if len(i) != 2 { - err = fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") - return "", nil, err + return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") } - return storageResult.CID, i[1].([]byte), err + return storageResult.CID, i[1].([]byte), nil } // RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number // This can retrun a non-canonical value func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) { - storageResult := new(ipldResult) + storageResult := new(nodeInfo) stateLeafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { return "", nil, err } + if storageResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())