From 1467ea392452323a38bc5b7a5b37d7ae425bcba5 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 9 Sep 2021 17:42:26 +0530 Subject: [PATCH 1/3] Fix state leaf key removed query. --- db/migrations/00014_create_stored_functions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index 0aa5c5f3..544cd715 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -18,14 +18,14 @@ $$ LANGUAGE SQL; -- +goose StatementEnd -- +goose StatementBegin --- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash +-- returns if a state node at the provided path was removed in the range >= the provided height and <= the provided block hash CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN AS $$ SELECT exists(SELECT 1 FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) WHERE state_path = path - AND block_number > height + AND block_number >= height AND block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = hash) From 32b4f565571fbcb2b713065115774b72802fc3fe Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 15 Sep 2021 17:19:08 +0530 Subject: [PATCH 2/3] Update state_cids and storgae_cids query to use node_type. --- pkg/eth/ipld_retriever.go | 40 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 7442756d..b91bac75 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -28,6 +28,10 @@ import ( ) const ( + // node type removed value. + // https://github.com/vulcanize/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34 + removedNode = 3 + RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) @@ -102,9 +106,7 @@ const ( INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, - data, - was_state_removed(state_path, block_number, $2) AS removed + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) @@ -115,12 +117,7 @@ const ( AND header_cids.id = (SELECT canonical_header_id(block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, - data, - was_state_removed(state_path, block_number, (SELECT block_hash - FROM eth.header_cids - WHERE block_number = $2 - LIMIT 1)) AS removed + RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) @@ -128,12 +125,7 @@ const ( AND block_number <= $2 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, - data, - was_storage_removed(storage_path, block_number, (SELECT block_hash - FROM eth.header_cids - WHERE block_number = $3 - LIMIT 1)) AS removed + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) @@ -143,9 +135,7 @@ const ( AND block_number <= $3 ORDER BY block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, - data, - was_storage_removed(storage_path, block_number, $3) AS removed + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type FROM eth.storage_cids INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) @@ -387,9 +377,9 @@ func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, } type nodeInfo struct { - CID string `db:"cid"` - Data []byte `db:"data"` - Removed bool `db:"removed"` + CID string `db:"cid"` + Data []byte `db:"data"` + NodeType int `db:"node_type"` } // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash @@ -400,7 +390,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { return "", nil, err } - if accountResult.Removed { + if accountResult.NodeType == removedNode { return "", []byte{}, nil } var i []interface{} @@ -421,7 +411,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil { return "", nil, err } - if accountResult.Removed { + if accountResult.NodeType == removedNode { return "", []byte{}, nil } var i []interface{} @@ -442,7 +432,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - if storageResult.Removed { + if storageResult.NodeType == removedNode { return "", []byte{}, []byte{}, nil } var i []interface{} @@ -464,7 +454,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil { return "", nil, err } - if storageResult.Removed { + if storageResult.NodeType == removedNode { return "", []byte{}, nil } var i []interface{} From f9abcfd33cb641591560cbb8a7fab7023e4ac04f Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Wed, 15 Sep 2021 17:29:45 +0530 Subject: [PATCH 3/3] Remove was_state_removed and was_storage_removed functions from DB migration. --- .../00014_create_stored_functions.sql | 37 ---------------- db/schema.sql | 42 ------------------- 2 files changed, 79 deletions(-) diff --git a/db/migrations/00014_create_stored_functions.sql b/db/migrations/00014_create_stored_functions.sql index 544cd715..1e9ad468 100644 --- a/db/migrations/00014_create_stored_functions.sql +++ b/db/migrations/00014_create_stored_functions.sql @@ -1,39 +1,4 @@ -- +goose Up --- +goose StatementBegin --- returns if a storage node at the provided path was removed in the range >= the provided height and <= the provided block hash -CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN -AS $$ -SELECT exists(SELECT 1 - FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE storage_path = path - AND block_number >= height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND storage_cids.node_type = 3 - LIMIT 1); -$$ LANGUAGE SQL; --- +goose StatementEnd - --- +goose StatementBegin --- returns if a state node at the provided path was removed in the range >= the provided height and <= the provided block hash -CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN -AS $$ -SELECT exists(SELECT 1 - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_path = path - AND block_number >= height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND state_cids.node_type = 3 - LIMIT 1); -$$ LANGUAGE SQL; --- +goose StatementEnd - -- +goose StatementBegin CREATE TYPE child_result AS ( has_child BOOLEAN, @@ -150,8 +115,6 @@ LANGUAGE 'plpgsql'; -- +goose StatementEnd -- +goose Down -DROP FUNCTION was_storage_removed; -DROP FUNCTION was_state_removed; DROP FUNCTION canonical_header_id; DROP FUNCTION canonical_header_from_array; DROP FUNCTION has_child; diff --git a/db/schema.sql b/db/schema.sql index f80542a5..05a4283d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -221,48 +221,6 @@ RETURN new_child_result; END $$; - --- --- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean - LANGUAGE sql - AS $$ -SELECT exists(SELECT 1 - FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE state_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND state_cids.node_type = 3 - LIMIT 1); -$$; - - --- --- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean - LANGUAGE sql - AS $$ -SELECT exists(SELECT 1 - FROM eth.storage_cids - INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) - WHERE storage_path = path - AND block_number > height - AND block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = hash) - AND storage_cids.node_type = 3 - LIMIT 1); -$$; - - -- -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: - --