cerc-io/ipld-eth-server
17
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

postgres functions to check if node was removed (e.g. a la EIP158) in a range; update backend to use these to retrieve state and storage leafs in single (albeit complex) SELECT query

Browse Source
This commit is contained in:
Ian Norden 2020-10-31 15:00:03 -05:00
parent 7c06d4b3a1
commit b128f894c4
3 changed files with 195 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
db/migrations/00018_create_check_if_removed_functions.sql Normal file
View File

@ -0,0 +1,37 @@
-- +goose Up
-- +goose StatementBegin
-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
AS $$
SELECT exists(SELECT *
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE storage_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND storage_cids.node_type = 3);
$$ LANGUAGE SQL;
-- +goose StatementEnd
-- +goose StatementBegin
-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
AS $$
SELECT exists(SELECT *
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND state_cids.node_type = 3);
$$ LANGUAGE SQL;
-- +goose StatementEnd
-- +goose Down
DROP FUNCTION was_storage_removed;
DROP FUNCTION was_state_removed;

39
db/schema.sql
View File

@ -100,6 +100,45 @@ CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint
$$;
--
-- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
--
CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
LANGUAGE sql
AS $$
SELECT exists(SELECT *
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE state_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND state_cids.node_type = 3);
$$;
--
-- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
--
CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
LANGUAGE sql
AS $$
SELECT exists(SELECT *
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE storage_path = path
AND block_number > height
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = hash)
AND storage_cids.node_type = 3);
$$;
SET default_tablespace = '';
SET default_table_access_method = heap;

246
pkg/eth/ipld_retriever.go
View File

@ -19,8 +19,6 @@ package eth
import (
"fmt"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
@ -32,122 +30,130 @@ import (
const (
RetrieveHeadersByHashesPgStr = `SELECT cid, data
FROM eth.header_cids
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
WHERE block_hash = ANY($1::VARCHAR(66)[])`
RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data
FROM eth.header_cids
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
WHERE block_number = $1`
RetrieveHeaderByHashPgStr = `SELECT cid, data
FROM eth.header_cids
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
WHERE block_hash = $1`
RetrieveUnclesByHashesPgStr = `SELECT cid, data
FROM eth.uncle_cids
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
WHERE block_hash = ANY($1::VARCHAR(66)[])`
RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks
WHERE uncle_cids.header_id = header_cids.id
AND uncle_cids.mh_key = blocks.key
AND block_hash = $1`
RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
FROM eth.uncle_cids
INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
WHERE block_hash = $1`
RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data
FROM eth.uncle_cids, eth.header_cids, public.blocks
WHERE uncle_cids.header_id = header_cids.id
AND uncle_cids.mh_key = blocks.key
AND block_number = $1`
FROM eth.uncle_cids
INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
WHERE block_number = $1`
RetrieveUncleByHashPgStr = `SELECT cid, data
FROM eth.uncle_cids
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
WHERE block_hash = $1`
RetrieveTransactionsByHashesPgStr = `SELECT cid, data
FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
FROM eth.transaction_cids, eth.header_cids, public.blocks
WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.mh_key = blocks.key
AND block_hash = $1`
FROM eth.transaction_cids
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE block_hash = $1`
RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data
FROM eth.transaction_cids, eth.header_cids, public.blocks
WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.mh_key = blocks.key
AND block_number = $1`
RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
FROM eth.transaction_cids
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE block_number = $1`
RetrieveTransactionByHashPgStr = `SELECT cid, data
FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = $1`
RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
FROM eth.receipt_cids, eth.transaction_cids, public.blocks
WHERE receipt_cids.mh_key = blocks.key
AND receipt_cids.tx_id = transaction_cids.id
AND tx_hash = ANY($1::VARCHAR(66)[])`
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND receipt_cids.mh_key = blocks.key
AND block_hash = $1`
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
WHERE block_hash = $1`
RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND receipt_cids.mh_key = blocks.key
AND block_number = $1`
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
WHERE block_number = $1`
RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids
WHERE receipt_cids.mh_key = blocks.key
AND receipt_cids.tx_id = transaction_cids.id
AND tx_hash = $1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data
FROM eth.state_cids, eth.header_cids, public.blocks
WHERE state_cids.header_id = header_cids.id
AND state_cids.mh_key = blocks.key
AND state_leaf_key = $1
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
WHERE tx_hash = $1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid,
data,
was_state_removed(state_path, block_number, $2) AS removed
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
WHERE state_leaf_key = $1
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = $2)
AND header_cids.id = (SELECT canonical_header(block_number))
ORDER BY block_number DESC
LIMIT 1`
RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data
FROM eth.state_cids, eth.header_cids, public.blocks
WHERE state_cids.header_id = header_cids.id
AND state_cids.mh_key = blocks.key
AND state_leaf_key = $1
AND block_number <= $2
ORDER BY block_number DESC
LIMIT 1`
RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data
FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND storage_cids.mh_key = blocks.key
AND state_leaf_key = $1
AND storage_leaf_key = $2
AND block_number <= $3
ORDER BY block_number DESC
LIMIT 1`
retrieveStorageInfoPgStr = `SELECT storage_cids.cid, data, storage_path, block_number
FROM eth.storage_cids, eth.state_cids, eth.header_cids, public.blocks
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND storage_cids.mh_key = blocks.key
AND state_leaf_key = $1
AND storage_leaf_key = $2
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = $3)
AND header_cids.id = (SELECT canonical_header(block_number))
ORDER BY block_number DESC
LIMIT 1`
wasNodeDeletedpgStr = `SELECT exists(SELECT *
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND storage_path = $1
AND block_number > $2
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = $3)
AND storage_cids.node_type = 3)`
RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid,
data,
was_state_removed(state_path, block_number, (SELECT block_hash
FROM eth.header_cids
WHERE block_number = $2
LIMIT 1)) AS removed
FROM eth.state_cids
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
WHERE state_leaf_key = $1
AND block_number <= $2
ORDER BY block_number DESC
LIMIT 1`
RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid,
data,
was_storage_removed(storage_path, block_number, (SELECT block_hash
FROM eth.header_cids
WHERE block_number = $3
LIMIT 1)) AS removed
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
WHERE state_leaf_key = $1
AND storage_leaf_key = $2
AND block_number <= $3
ORDER BY block_number DESC
LIMIT 1`
RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid,
data,
was_storage_removed(storage_path, block_number, $3) AS removed
FROM eth.storage_cids
INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
WHERE state_leaf_key = $1
AND storage_leaf_key = $2
AND block_number <= (SELECT block_number
FROM eth.header_cids
WHERE block_hash = $3)
AND header_cids.id = (SELECT canonical_header(block_number))
ORDER BY block_number DESC
LIMIT 1`
)
type ipldResult struct {
@ -370,14 +376,23 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte,
return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
}
type nodeInfo struct {
CID string `db:"cid"`
Data []byte `db:"data"`
Removed bool `db:"removed"`
}
// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
// TODO: ensure this handles deleted accounts appropriately
func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
accountResult := new(ipldResult)
accountResult := new(nodeInfo)
leafKey := crypto.Keccak256Hash(address.Bytes())
if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
return "", nil, err
}
if accountResult.Removed {
return "", []byte{}, nil
}
var i []interface{}
if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
@ -391,11 +406,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
// RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
// This can return a non-canonical account
func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) {
accountResult := new(ipldResult)
accountResult := new(nodeInfo)
leafKey := crypto.Keccak256Hash(address.Bytes())
if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
return "", nil, err
}
if accountResult.Removed {
return "", []byte{}, nil
}
var i []interface{}
if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
@ -406,42 +424,14 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad
return accountResult.CID, i[1].([]byte), nil
}
type storageInfo struct {
CID string `db:"cid"`
Data []byte `db:"data"`
Path []byte `db:"storage_path"`
BlockNumber uint64 `db:"block_number"`
}
// RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash
func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) {
// Begin tx
tx, err := r.db.Beginx()
if err != nil {
return "", nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
storageResult := new(storageInfo)
storageResult := new(nodeInfo)
stateLeafKey := crypto.Keccak256Hash(address.Bytes())
if err := tx.Get(storageResult, retrieveStorageInfoPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil {
if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil {
return "", nil, err
}
deleted := false
if err := tx.Get(&deleted, wasNodeDeletedpgStr, storageResult.Path, storageResult.BlockNumber, hash.Hex()); err != nil {
return "", nil, err
}
if deleted {
if storageResult.Removed {
return "", []byte{}, nil
}
var i []interface{}
@ -450,20 +440,22 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(addr
return "", nil, err
}
if len(i) != 2 {
err = fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
return "", nil, err
return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
}
return storageResult.CID, i[1].([]byte), err
return storageResult.CID, i[1].([]byte), nil
}
// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number
// This can retrun a non-canonical value
func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) {
storageResult := new(ipldResult)
storageResult := new(nodeInfo)
stateLeafKey := crypto.Keccak256Hash(address.Bytes())
if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil {
return "", nil, err
}
if storageResult.Removed {
return "", []byte{}, nil
}
var i []interface{}
if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())