From b128f894c450e3be19dcf6a3d655d1fcf0e03bdb Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sat, 31 Oct 2020 15:00:03 -0500 Subject: [PATCH] postgres functions to check if node was removed (e.g. a la EIP158) in a range; update backend to use these to retrieve state and storage leafs in single (albeit complex) SELECT query --- ...0018_create_check_if_removed_functions.sql | 37 +++ db/schema.sql | 39 +++ pkg/eth/ipld_retriever.go | 246 +++++++++--------- 3 files changed, 195 insertions(+), 127 deletions(-) create mode 100644 db/migrations/00018_create_check_if_removed_functions.sql diff --git a/db/migrations/00018_create_check_if_removed_functions.sql b/db/migrations/00018_create_check_if_removed_functions.sql new file mode 100644 index 00000000..9a7e42d3 --- /dev/null +++ b/db/migrations/00018_create_check_if_removed_functions.sql @@ -0,0 +1,37 @@ +-- +goose Up +-- +goose StatementBegin +-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash +CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN +AS $$ + SELECT exists(SELECT * + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE storage_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND storage_cids.node_type = 3); +$$ LANGUAGE SQL; +-- +goose StatementEnd + +-- +goose StatementBegin +-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash +CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN +AS $$ + SELECT exists(SELECT * + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND state_cids.node_type = 3); +$$ LANGUAGE SQL; +-- +goose StatementEnd + +-- +goose Down +DROP FUNCTION was_storage_removed; +DROP FUNCTION was_state_removed; \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 02e92f20..91be03f0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -100,6 +100,45 @@ CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint $$; +-- +-- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT exists(SELECT * + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE state_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND state_cids.node_type = 3); +$$; + + +-- +-- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT exists(SELECT * + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE storage_path = path + AND block_number > height + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + AND storage_cids.node_type = 3); +$$; + + SET default_tablespace = ''; SET default_table_access_method = heap; diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 487058cd..8418bfe6 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,8 +19,6 @@ package eth import ( "fmt" - "github.com/vulcanize/ipld-eth-server/pkg/shared" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -32,122 +30,130 @@ import ( const ( RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data + FROM eth.header_cids + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_number = $1` RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) WHERE block_hash = $1` RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = ANY($1::VARCHAR(66)[])` - RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks - WHERE uncle_cids.header_id = header_cids.id - AND uncle_cids.mh_key = blocks.key - AND block_hash = $1` + RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data - FROM eth.uncle_cids, eth.header_cids, public.blocks - WHERE uncle_cids.header_id = header_cids.id - AND uncle_cids.mh_key = blocks.key - AND block_number = $1` + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + WHERE block_number = $1` RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) WHERE block_hash = $1` RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids, eth.header_cids, public.blocks - WHERE transaction_cids.header_id = header_cids.id - AND transaction_cids.mh_key = blocks.key - AND block_hash = $1` + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids, eth.header_cids, public.blocks - WHERE transaction_cids.header_id = header_cids.id - AND transaction_cids.mh_key = blocks.key - AND block_number = $1` - RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + WHERE block_number = $1` + RetrieveTransactionByHashPgStr = `SELECT cid, data + FROM eth.transaction_cids + INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, public.blocks - WHERE receipt_cids.mh_key = blocks.key - AND receipt_cids.tx_id = transaction_cids.id - AND tx_hash = ANY($1::VARCHAR(66)[])` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id - AND receipt_cids.mh_key = blocks.key - AND block_hash = $1` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE block_hash = $1` RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks - WHERE receipt_cids.tx_id = transaction_cids.id - AND transaction_cids.header_id = header_cids.id - AND receipt_cids.mh_key = blocks.key - AND block_number = $1` + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE block_number = $1` RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data - FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids - WHERE receipt_cids.mh_key = blocks.key - AND receipt_cids.tx_id = transaction_cids.id - AND tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data - FROM eth.state_cids, eth.header_cids, public.blocks - WHERE state_cids.header_id = header_cids.id - AND state_cids.mh_key = blocks.key - AND state_leaf_key = $1 + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) + INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + WHERE tx_hash = $1` + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, + data, + was_state_removed(state_path, block_number, $2) AS removed + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) AND header_cids.id = (SELECT canonical_header(block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data - FROM eth.state_cids, eth.header_cids, public.blocks - WHERE state_cids.header_id = header_cids.id - AND state_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND block_number <= $2 - ORDER BY block_number DESC - LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= $3 - ORDER BY block_number DESC - LIMIT 1` - retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number - FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks - WHERE storage_cids.state_id = state_cids.id - AND state_cids.header_id = header_cids.id - AND storage_cids.mh_key = blocks.key - AND state_leaf_key = $1 - AND storage_leaf_key = $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND header_cids.id = (SELECT canonical_header(block_number)) - ORDER BY block_number DESC - LIMIT 1` - wasNodeDeletedpgStr = `SELECT exists(SELECT * - FROM eth.storage_cids, eth.state_cids, eth.header_cids - WHERE storage_cids.state_id = state_cids.id - AND storage_path = $1 - AND block_number > $2 - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $3) - AND storage_cids.node_type = 3)` + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, + data, + was_state_removed(state_path, block_number, (SELECT block_hash + FROM eth.header_cids + WHERE block_number = $2 + LIMIT 1)) AS removed + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND block_number <= $2 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, + data, + was_storage_removed(storage_path, block_number, (SELECT block_hash + FROM eth.header_cids + WHERE block_number = $3 + LIMIT 1)) AS removed + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= $3 + ORDER BY block_number DESC + LIMIT 1` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, + data, + was_storage_removed(storage_path, block_number, $3) AS removed + FROM eth.storage_cids + INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) + WHERE state_leaf_key = $1 + AND storage_leaf_key = $2 + AND block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $3) + AND header_cids.id = (SELECT canonical_header(block_number)) + ORDER BY block_number DESC + LIMIT 1` ) type ipldResult struct { @@ -370,14 +376,23 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) } +type nodeInfo struct { + CID string `db:"cid"` + Data []byte `db:"data"` + Removed bool `db:"removed"` +} + // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash // TODO: ensure this handles deleted accounts appropriately func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { - accountResult := new(ipldResult) + accountResult := new(nodeInfo) leafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } + if accountResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -391,11 +406,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number // This can return a non-canonical account func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) { - accountResult := new(ipldResult) + accountResult := new(nodeInfo) leafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { return "", nil, err } + if accountResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) @@ -406,42 +424,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad return accountResult.CID, i[1].([]byte), nil } -type storageInfo struct { - CID string `db:"cid"` - Data []byte `db:"data"` - Path []byte `db:"storage_path"` - BlockNumber uint64 `db:"block_number"` -} - // RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) { - // Begin tx - tx, err := r.db.Beginx() - if err != nil { - return "", nil, err - } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - - storageResult := new(storageInfo) + storageResult := new(nodeInfo) stateLeafKey := crypto.Keccak256Hash(address.Bytes()) - if err := tx.Get(storageResult, retrieveStorageInfoPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } - - deleted := false - if err := tx.Get(&deleted, wasNodeDeletedpgStr, storageResult.Path, storageResult.BlockNumber, hash.Hex()); err != nil { - return "", nil, err - } - if deleted { + if storageResult.Removed { return "", []byte{}, nil } var i []interface{} @@ -450,20 +440,22 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(addr return "", nil, err } if len(i) != 2 { - err = fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") - return "", nil, err + return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") } - return storageResult.CID, i[1].([]byte), err + return storageResult.CID, i[1].([]byte), nil } // RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number // This can retrun a non-canonical value func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) { - storageResult := new(ipldResult) + storageResult := new(nodeInfo) stateLeafKey := crypto.Keccak256Hash(address.Bytes()) if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { return "", nil, err } + if storageResult.Removed { + return "", []byte{}, nil + } var i []interface{} if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())