From 2bf5c821500e849035af2649a2841490b053939b Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 8 Aug 2022 12:40:47 -0500 Subject: [PATCH 1/6] replace btree with hash index where it makes sense --- db/migrations/00014_create_cid_indexes.sql | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/db/migrations/00014_create_cid_indexes.sql b/db/migrations/00014_create_cid_indexes.sql index 662d040..a360017 100644 --- a/db/migrations/00014_create_cid_indexes.sql +++ b/db/migrations/00014_create_cid_indexes.sql @@ -3,7 +3,7 @@ CREATE INDEX header_block_number_index ON eth.header_cids USING brin (block_number); CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid, block_number); CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree (mh_key, block_number); -CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root); +CREATE INDEX state_root_index ON eth.header_cids USING hash (state_root); CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp); -- uncle indexes @@ -13,19 +13,19 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id); -- transaction indexes CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number); -CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id); +CREATE INDEX tx_header_id_index ON eth.transaction_cids USING hash (header_id); CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number); CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number); -CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst); -CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); +CREATE INDEX tx_dst_index ON eth.transaction_cids USING hash (dst); +CREATE INDEX tx_src_index ON eth.transaction_cids USING hash (src); CREATE INDEX tx_data_index ON eth.transaction_cids USING hash (tx_data); -- receipt indexes CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number); -CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id); +CREATE INDEX rct_header_id_index ON eth.receipt_cids USING hash (header_id); CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number); -CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); +CREATE INDEX rct_contract_index ON eth.receipt_cids USING hash (contract); CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash); -- state node indexes @@ -33,7 +33,7 @@ CREATE INDEX state_block_number_index ON eth.state_cids USING brin (block_number CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key); CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid); CREATE INDEX state_mh_block_number_index ON eth.state_cids USING btree (mh_key, block_number); -CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id); +CREATE INDEX state_header_id_index ON eth.state_cids USING hash (header_id); CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type); -- storage node indexes @@ -42,29 +42,29 @@ CREATE INDEX storage_state_path_index ON eth.storage_cids USING btree (state_pat CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key); CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid); CREATE INDEX storage_mh_block_number_index ON eth.storage_cids USING btree (mh_key, block_number); -CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id); +CREATE INDEX storage_header_id_index ON eth.storage_cids USING hash (header_id); CREATE INDEX storage_node_type_index ON eth.storage_cids USING btree (node_type); -- state accounts indexes CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_number); -CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id); -CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root); +CREATE INDEX account_header_id_index ON eth.state_accounts USING hash (header_id); +CREATE INDEX account_storage_root_index ON eth.state_accounts USING hash (storage_root); -- access list indexes CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING brin (block_number); -CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address); +CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING hash (address); CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gin (storage_keys); -- log indexes CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number); -CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id); +CREATE INDEX log_header_id_index ON eth.log_cids USING hash (header_id); CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number); CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid); -CREATE INDEX log_address_index ON eth.log_cids USING btree (address); -CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0); -CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1); -CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2); -CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); +CREATE INDEX log_address_index ON eth.log_cids USING hash (address); +CREATE INDEX log_topic0_index ON eth.log_cids USING hash (topic0); +CREATE INDEX log_topic1_index ON eth.log_cids USING hash (topic1); +CREATE INDEX log_topic2_index ON eth.log_cids USING hash (topic2); +CREATE INDEX log_topic3_index ON eth.log_cids USING hash (topic3); CREATE INDEX log_data_index ON eth.log_cids USING hash (log_data); -- +goose Down From 42f46fc397d51e8607581c170e062e63fee1a930 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 8 Aug 2022 12:42:54 -0500 Subject: [PATCH 2/6] update schema.sql --- schema.sql | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/schema.sql b/schema.sql index cf1f53b..447706d 100644 --- a/schema.sql +++ b/schema.sql @@ -519,7 +519,7 @@ CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING br -- Name: access_list_element_address_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address); +CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING hash (address); -- @@ -540,14 +540,14 @@ CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_ -- Name: account_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id); +CREATE INDEX account_header_id_index ON eth.state_accounts USING hash (header_id); -- -- Name: account_storage_root_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root); +CREATE INDEX account_storage_root_index ON eth.state_accounts USING hash (storage_root); -- @@ -575,7 +575,7 @@ CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree -- Name: log_address_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_address_index ON eth.log_cids USING btree (address); +CREATE INDEX log_address_index ON eth.log_cids USING hash (address); -- @@ -603,7 +603,7 @@ CREATE INDEX log_data_index ON eth.log_cids USING hash (log_data); -- Name: log_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id); +CREATE INDEX log_header_id_index ON eth.log_cids USING hash (header_id); -- @@ -617,28 +617,28 @@ CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh -- Name: log_topic0_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0); +CREATE INDEX log_topic0_index ON eth.log_cids USING hash (topic0); -- -- Name: log_topic1_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1); +CREATE INDEX log_topic1_index ON eth.log_cids USING hash (topic1); -- -- Name: log_topic2_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2); +CREATE INDEX log_topic2_index ON eth.log_cids USING hash (topic2); -- -- Name: log_topic3_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); +CREATE INDEX log_topic3_index ON eth.log_cids USING hash (topic3); -- @@ -659,14 +659,14 @@ CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_h -- Name: rct_contract_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); +CREATE INDEX rct_contract_index ON eth.receipt_cids USING hash (contract); -- -- Name: rct_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id); +CREATE INDEX rct_header_id_index ON eth.receipt_cids USING hash (header_id); -- @@ -701,7 +701,7 @@ CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid); -- Name: state_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id); +CREATE INDEX state_header_id_index ON eth.state_cids USING hash (header_id); -- @@ -729,7 +729,7 @@ CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type); -- Name: state_root_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root); +CREATE INDEX state_root_index ON eth.header_cids USING hash (state_root); -- @@ -750,7 +750,7 @@ CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid); -- Name: storage_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id); +CREATE INDEX storage_header_id_index ON eth.storage_cids USING hash (header_id); -- @@ -813,14 +813,14 @@ CREATE INDEX tx_data_index ON eth.transaction_cids USING hash (tx_data); -- Name: tx_dst_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst); +CREATE INDEX tx_dst_index ON eth.transaction_cids USING hash (dst); -- -- Name: tx_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id); +CREATE INDEX tx_header_id_index ON eth.transaction_cids USING hash (header_id); -- @@ -834,7 +834,7 @@ CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_ke -- Name: tx_src_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); +CREATE INDEX tx_src_index ON eth.transaction_cids USING hash (src); -- From 475ead282b663249c77b65710248c63433aa0628 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 15 Aug 2022 11:11:19 -0500 Subject: [PATCH 3/6] remove known_gaps table --- db/migrations/00019_create_known_gaps_table.sql | 10 ---------- ...date_db_version.sql => 00019_update_db_version.sql} | 0 ...ypertables.sql => 00020_convert_to_hypertables.sql} | 0 ...functions.sql => 00021_create_stored_functions.sql} | 0 4 files changed, 10 deletions(-) delete mode 100644 db/migrations/00019_create_known_gaps_table.sql rename db/migrations/{00020_update_db_version.sql => 00019_update_db_version.sql} (100%) rename db/migrations/{00021_convert_to_hypertables.sql => 00020_convert_to_hypertables.sql} (100%) rename db/migrations/{00022_create_stored_functions.sql => 00021_create_stored_functions.sql} (100%) diff --git a/db/migrations/00019_create_known_gaps_table.sql b/db/migrations/00019_create_known_gaps_table.sql deleted file mode 100644 index 4a7c730..0000000 --- a/db/migrations/00019_create_known_gaps_table.sql +++ /dev/null @@ -1,10 +0,0 @@ --- +goose Up -CREATE TABLE eth_meta.known_gaps ( - starting_block_number bigint PRIMARY KEY, - ending_block_number bigint, - checked_out boolean, - processing_key bigint -); - --- +goose Down -DROP TABLE eth_meta.known_gaps; diff --git a/db/migrations/00020_update_db_version.sql b/db/migrations/00019_update_db_version.sql similarity index 100% rename from db/migrations/00020_update_db_version.sql rename to db/migrations/00019_update_db_version.sql diff --git a/db/migrations/00021_convert_to_hypertables.sql b/db/migrations/00020_convert_to_hypertables.sql similarity index 100% rename from db/migrations/00021_convert_to_hypertables.sql rename to db/migrations/00020_convert_to_hypertables.sql diff --git a/db/migrations/00022_create_stored_functions.sql b/db/migrations/00021_create_stored_functions.sql similarity index 100% rename from db/migrations/00022_create_stored_functions.sql rename to db/migrations/00021_create_stored_functions.sql From 701f9c2729b2e285d43b02b15715bb0410b6a770 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 15 Aug 2022 11:15:48 -0500 Subject: [PATCH 4/6] update schema.sql --- schema.sql | 448 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 388 insertions(+), 60 deletions(-) diff --git a/schema.sql b/schema.sql index 447706d..7ef2304 100644 --- a/schema.sql +++ b/schema.sql @@ -16,6 +16,20 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: timescaledb; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS timescaledb WITH SCHEMA public; + + +-- +-- Name: EXTENSION timescaledb; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION timescaledb IS 'Enables scalable inserts and complex queries for time-series data'; + + -- -- Name: eth; Type: SCHEMA; Schema: -; Owner: - -- @@ -30,6 +44,87 @@ CREATE SCHEMA eth; CREATE SCHEMA eth_meta; +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: header_cids; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.header_cids ( + block_number bigint NOT NULL, + block_hash character varying(66) NOT NULL, + parent_hash character varying(66) NOT NULL, + cid text NOT NULL, + td numeric NOT NULL, + node_id character varying(128) NOT NULL, + reward numeric NOT NULL, + state_root character varying(66) NOT NULL, + tx_root character varying(66) NOT NULL, + receipt_root character varying(66) NOT NULL, + uncles_hash character varying(66) NOT NULL, + bloom bytea NOT NULL, + "timestamp" bigint NOT NULL, + mh_key text NOT NULL, + times_validated integer DEFAULT 1 NOT NULL, + coinbase character varying(66) NOT NULL +); + + +-- +-- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids'; + + +-- +-- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: - +-- + +COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID'; + + +-- +-- Name: child_result; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.child_result AS ( + has_child boolean, + children eth.header_cids[] +); + + +-- +-- Name: state_node_result; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.state_node_result AS ( + data bytea, + state_leaf_key character varying(66), + cid text, + state_path bytea, + node_type integer, + mh_key text +); + + +-- +-- Name: storage_node_result; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.storage_node_result AS ( + data bytea, + state_path bytea, + storage_leaf_key character varying(66), + cid text, + storage_path bytea, + node_type integer, + mh_key text +); + + -- -- Name: graphql_subscription(); Type: FUNCTION; Schema: eth; Owner: - -- @@ -93,9 +188,223 @@ END; $$; -SET default_tablespace = ''; +-- +-- Name: canonical_header_from_array(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids + LANGUAGE plpgsql + AS $$ +DECLARE + canonical_header eth.header_cids; + canonical_child eth.header_cids; + header eth.header_cids; + current_child_result child_result; + child_headers eth.header_cids[]; + current_header_with_child eth.header_cids; + has_children_count INT DEFAULT 0; +BEGIN + -- for each header in the provided set + FOREACH header IN ARRAY headers + LOOP + -- check if it has any children + current_child_result = has_child(header.block_hash, header.block_number); + IF current_child_result.has_child THEN + -- if it does, take note + has_children_count = has_children_count + 1; + current_header_with_child = header; + -- and add the children to the growing set of child headers + child_headers = array_cat(child_headers, current_child_result.children); + END IF; + END LOOP; + -- if none of the headers had children, none is more canonical than the other + IF has_children_count = 0 THEN + -- return the first one selected + SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1; + -- if only one header had children, it can be considered the heaviest/canonical header of the set + ELSIF has_children_count = 1 THEN + -- return the only header with a child + canonical_header = current_header_with_child; + -- if there are multiple headers with children + ELSE + -- find the canonical header from the child set + canonical_child = canonical_header_from_array(child_headers); + -- the header that is parent to this header, is the canonical header at this level + SELECT * INTO canonical_header FROM unnest(headers) + WHERE block_hash = canonical_child.parent_hash; + END IF; + RETURN canonical_header; +END +$$; + + +-- +-- Name: canonical_header_hash(bigint); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.canonical_header_hash(height bigint) RETURNS character varying + LANGUAGE plpgsql + AS $$ +DECLARE + canonical_header eth.header_cids; + headers eth.header_cids[]; + header_count INT; + temp_header eth.header_cids; +BEGIN + -- collect all headers at this height + FOR temp_header IN + SELECT * FROM eth.header_cids WHERE block_number = height + LOOP + headers = array_append(headers, temp_header); + END LOOP; + -- count the number of headers collected + header_count = array_length(headers, 1); + -- if we have less than 1 header, return NULL + IF header_count IS NULL OR header_count < 1 THEN + RETURN NULL; + -- if we have one header, return its hash + ELSIF header_count = 1 THEN + RETURN headers[1].block_hash; + -- if we have multiple headers we need to determine which one is canonical + ELSE + canonical_header = canonical_header_from_array(headers); + RETURN canonical_header.block_hash; + END IF; +END +$$; + + +-- +-- Name: has_child(character varying, bigint); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.has_child(hash character varying, height bigint) RETURNS public.child_result + LANGUAGE plpgsql + AS $$ +DECLARE + child_height INT; + temp_child eth.header_cids; + new_child_result child_result; +BEGIN + child_height = height + 1; + -- short circuit if there are no children + SELECT exists(SELECT 1 + FROM eth.header_cids + WHERE parent_hash = hash + AND block_number = child_height + LIMIT 1) + INTO new_child_result.has_child; + -- collect all the children for this header + IF new_child_result.has_child THEN + FOR temp_child IN + SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height + LOOP + new_child_result.children = array_append(new_child_result.children, temp_child); + END LOOP; + END IF; + RETURN new_child_result; +END +$$; + + +-- +-- Name: state_snapshot(bigint, bigint); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.state_snapshot(starting_height bigint, ending_height bigint) RETURNS void + LANGUAGE plpgsql + AS $$ +DECLARE + canonical_hash VARCHAR(66); + results state_node_result[]; +BEGIN + -- get the canonical hash for the header at ending_height + canonical_hash = canonical_header_hash(ending_height); + IF canonical_hash IS NULL THEN + RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height; + END IF; + -- select all of the state nodes for this snapshot: the latest state node record at every unique path + SELECT ARRAY (SELECT DISTINCT ON (state_path) ROW (blocks.data, state_cids.state_leaf_key, state_cids.cid, state_cids.state_path, + state_cids.node_type, state_cids.mh_key) + FROM eth.state_cids + INNER JOIN public.blocks + ON (state_cids.mh_key, state_cids.block_number) = (blocks.key, blocks.block_number) + WHERE state_cids.block_number BETWEEN starting_height AND ending_height + ORDER BY state_path, state_cids.block_number DESC) + INTO results; + -- from the set returned above, insert public.block records at the ending_height block number + INSERT INTO public.blocks (block_number, key, data) + SELECT ending_height, r.mh_key, r.data + FROM unnest(results) r; + -- from the set returned above, insert eth.state_cids records at the ending_height block number + -- anchoring all the records to the canonical header found at ending_height + INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) + SELECT ending_height, canonical_hash, r.state_leaf_key, r.cid, r.state_path, r.node_type, false, r.mh_key + FROM unnest(results) r + ON CONFLICT (state_path, header_id, block_number) DO NOTHING; +END +$$; + + +-- +-- Name: storage_snapshot(bigint, bigint); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.storage_snapshot(starting_height bigint, ending_height bigint) RETURNS void + LANGUAGE plpgsql + AS $$ +DECLARE + canonical_hash VARCHAR(66); + results storage_node_result[]; +BEGIN + -- get the canonical hash for the header at ending_height + SELECT canonical_header_hash(ending_height) INTO canonical_hash; + IF canonical_hash IS NULL THEN + RAISE EXCEPTION 'cannot create state snapshot, no header can be found at height %', ending_height; + END IF; + -- select all of the storage nodes for this snapshot: the latest storage node record at every unique state leaf key + SELECT ARRAY (SELECT DISTINCT ON (state_leaf_key, storage_path) ROW (blocks.data, storage_cids.state_path, storage_cids.storage_leaf_key, + storage_cids.cid, storage_cids.storage_path, storage_cids.node_type, storage_cids.mh_key) + FROM eth.storage_cids + INNER JOIN public.blocks + ON (storage_cids.mh_key, storage_cids.block_number) = (blocks.key, blocks.block_number) + INNER JOIN eth.state_cids + ON (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) + WHERE storage_cids.block_number BETWEEN starting_height AND ending_height + ORDER BY state_leaf_key, storage_path, storage_cids.state_path, storage_cids.block_number DESC) + INTO results; + -- from the set returned above, insert public.block records at the ending_height block number + INSERT INTO public.blocks (block_number, key, data) + SELECT ending_height, r.mh_key, r.data + FROM unnest(results) r; + -- from the set returned above, insert eth.state_cids records at the ending_height block number + -- anchoring all the records to the canonical header found at ending_height + INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, + node_type, diff, mh_key) + SELECT ending_height, canonical_hash, r.state_path, r.storage_leaf_key, r.cid, r.storage_path, r.node_type, false, r.mh_key + FROM unnest(results) r + ON CONFLICT (storage_path, state_path, header_id, block_number) DO NOTHING; +END +$$; + + +-- +-- Name: was_state_leaf_removed(character varying, character varying); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.was_state_leaf_removed(key character varying, hash character varying) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT state_cids.node_type = 3 + FROM eth.state_cids + INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) + WHERE state_leaf_key = key + AND state_cids.block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = hash) + ORDER BY state_cids.block_number DESC LIMIT 1; +$$; -SET default_table_access_method = heap; -- -- Name: access_list_elements; Type: TABLE; Schema: eth; Owner: - @@ -110,44 +419,6 @@ CREATE TABLE eth.access_list_elements ( ); --- --- Name: header_cids; Type: TABLE; Schema: eth; Owner: - --- - -CREATE TABLE eth.header_cids ( - block_number bigint NOT NULL, - block_hash character varying(66) NOT NULL, - parent_hash character varying(66) NOT NULL, - cid text NOT NULL, - td numeric NOT NULL, - node_id character varying(128) NOT NULL, - reward numeric NOT NULL, - state_root character varying(66) NOT NULL, - tx_root character varying(66) NOT NULL, - receipt_root character varying(66) NOT NULL, - uncles_hash character varying(66) NOT NULL, - bloom bytea NOT NULL, - "timestamp" bigint NOT NULL, - mh_key text NOT NULL, - times_validated integer DEFAULT 1 NOT NULL, - coinbase character varying(66) NOT NULL -); - - --- --- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: - --- - -COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids'; - - --- --- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: - --- - -COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID'; - - -- -- Name: log_cids; Type: TABLE; Schema: eth; Owner: - -- @@ -275,18 +546,6 @@ CREATE TABLE eth.uncle_cids ( ); --- --- Name: known_gaps; Type: TABLE; Schema: eth_meta; Owner: - --- - -CREATE TABLE eth_meta.known_gaps ( - starting_block_number bigint NOT NULL, - ending_block_number bigint, - checked_out boolean, - processing_key bigint -); - - -- -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - -- @@ -460,14 +719,6 @@ ALTER TABLE ONLY eth.uncle_cids ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash, block_number); --- --- Name: known_gaps known_gaps_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - --- - -ALTER TABLE ONLY eth_meta.known_gaps - ADD CONSTRAINT known_gaps_pkey PRIMARY KEY (starting_block_number); - - -- -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - -- @@ -858,6 +1109,13 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id); CREATE UNIQUE INDEX uncle_mh_block_number_index ON eth.uncle_cids USING btree (mh_key, block_number); +-- +-- Name: blocks_block_number_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX blocks_block_number_idx ON public.blocks USING btree (block_number DESC); + + -- -- Name: access_list_elements trg_eth_access_list_elements; Type: TRIGGER; Schema: eth; Owner: - -- @@ -921,6 +1179,76 @@ CREATE TRIGGER trg_eth_transaction_cids AFTER INSERT ON eth.transaction_cids FOR CREATE TRIGGER trg_eth_uncle_cids AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription(); +-- +-- Name: access_list_elements ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.access_list_elements FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: header_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.header_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: log_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.log_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: receipt_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: state_accounts ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_accounts FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: state_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: storage_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: transaction_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: uncle_cids ts_insert_blocker; Type: TRIGGER; Schema: eth; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + +-- +-- Name: blocks ts_insert_blocker; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON public.blocks FOR EACH ROW EXECUTE FUNCTION _timescaledb_internal.insert_blocker(); + + -- -- PostgreSQL database dump complete -- From 7f8247cb4f817b140e0dbcfb289d649521077628 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 30 Aug 2022 12:05:51 -0500 Subject: [PATCH 5/6] revert hash indexes; add eth probes tables --- .../00014_create_pending_tx_table.sql | 8 ++ ...dexes.sql => 00015_create_cid_indexes.sql} | 38 ++++---- ...=> 00016_create_postgraphile_triggers.sql} | 0 ....sql => 00017_create_db_version_table.sql} | 0 ...a.sql => 00018_create_eth_meta_schema.sql} | 0 ... 00019_create_watched_addresses_table.sql} | 0 db/migrations/00019_update_db_version.sql | 6 -- .../00022_create_probes_meta_tables.sql | 97 +++++++++++++++++++ db/migrations/00023_update_db_version.sql | 6 ++ 9 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 db/migrations/00014_create_pending_tx_table.sql rename db/migrations/{00014_create_cid_indexes.sql => 00015_create_cid_indexes.sql} (80%) rename db/migrations/{00015_create_postgraphile_triggers.sql => 00016_create_postgraphile_triggers.sql} (100%) rename db/migrations/{00016_create_db_version_table.sql => 00017_create_db_version_table.sql} (100%) rename db/migrations/{00017_create_eth_meta_schema.sql => 00018_create_eth_meta_schema.sql} (100%) rename db/migrations/{00018_create_watched_addresses_table.sql => 00019_create_watched_addresses_table.sql} (100%) delete mode 100644 db/migrations/00019_update_db_version.sql create mode 100644 db/migrations/00022_create_probes_meta_tables.sql create mode 100644 db/migrations/00023_update_db_version.sql diff --git a/db/migrations/00014_create_pending_tx_table.sql b/db/migrations/00014_create_pending_tx_table.sql new file mode 100644 index 0000000..1a8c7b3 --- /dev/null +++ b/db/migrations/00014_create_pending_tx_table.sql @@ -0,0 +1,8 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS eth.pending_txs ( + tx_hash VARCHAR(66) NOT NULL PRIMARY KEY, + mh_key TEXT NOT NULL +); + +-- +goose Down +DROP TABLE eth.pending_txs; diff --git a/db/migrations/00014_create_cid_indexes.sql b/db/migrations/00015_create_cid_indexes.sql similarity index 80% rename from db/migrations/00014_create_cid_indexes.sql rename to db/migrations/00015_create_cid_indexes.sql index a360017..23d139e 100644 --- a/db/migrations/00014_create_cid_indexes.sql +++ b/db/migrations/00015_create_cid_indexes.sql @@ -3,7 +3,7 @@ CREATE INDEX header_block_number_index ON eth.header_cids USING brin (block_number); CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid, block_number); CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree (mh_key, block_number); -CREATE INDEX state_root_index ON eth.header_cids USING hash (state_root); +CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root); CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp); -- uncle indexes @@ -13,19 +13,19 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id); -- transaction indexes CREATE INDEX tx_block_number_index ON eth.transaction_cids USING brin (block_number); -CREATE INDEX tx_header_id_index ON eth.transaction_cids USING hash (header_id); +CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id); CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number); CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_key, block_number); -CREATE INDEX tx_dst_index ON eth.transaction_cids USING hash (dst); -CREATE INDEX tx_src_index ON eth.transaction_cids USING hash (src); -CREATE INDEX tx_data_index ON eth.transaction_cids USING hash (tx_data); +CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst); +CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); +CREATE INDEX tx_data_index ON eth.transaction_cids USING btree (tx_data); -- receipt indexes CREATE INDEX rct_block_number_index ON eth.receipt_cids USING brin (block_number); -CREATE INDEX rct_header_id_index ON eth.receipt_cids USING hash (header_id); +CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id); CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); CREATE INDEX rct_leaf_mh_block_number_index ON eth.receipt_cids USING btree (leaf_mh_key, block_number); -CREATE INDEX rct_contract_index ON eth.receipt_cids USING hash (contract); +CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash); -- state node indexes @@ -33,7 +33,7 @@ CREATE INDEX state_block_number_index ON eth.state_cids USING brin (block_number CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key); CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid); CREATE INDEX state_mh_block_number_index ON eth.state_cids USING btree (mh_key, block_number); -CREATE INDEX state_header_id_index ON eth.state_cids USING hash (header_id); +CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id); CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type); -- storage node indexes @@ -42,30 +42,30 @@ CREATE INDEX storage_state_path_index ON eth.storage_cids USING btree (state_pat CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key); CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid); CREATE INDEX storage_mh_block_number_index ON eth.storage_cids USING btree (mh_key, block_number); -CREATE INDEX storage_header_id_index ON eth.storage_cids USING hash (header_id); +CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id); CREATE INDEX storage_node_type_index ON eth.storage_cids USING btree (node_type); -- state accounts indexes CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_number); -CREATE INDEX account_header_id_index ON eth.state_accounts USING hash (header_id); -CREATE INDEX account_storage_root_index ON eth.state_accounts USING hash (storage_root); +CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id); +CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root); -- access list indexes CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING brin (block_number); -CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING hash (address); +CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address); CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gin (storage_keys); -- log indexes CREATE INDEX log_block_number_index ON eth.log_cids USING brin (block_number); -CREATE INDEX log_header_id_index ON eth.log_cids USING hash (header_id); +CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id); CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh_key, block_number); CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid); -CREATE INDEX log_address_index ON eth.log_cids USING hash (address); -CREATE INDEX log_topic0_index ON eth.log_cids USING hash (topic0); -CREATE INDEX log_topic1_index ON eth.log_cids USING hash (topic1); -CREATE INDEX log_topic2_index ON eth.log_cids USING hash (topic2); -CREATE INDEX log_topic3_index ON eth.log_cids USING hash (topic3); -CREATE INDEX log_data_index ON eth.log_cids USING hash (log_data); +CREATE INDEX log_address_index ON eth.log_cids USING btree (address); +CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0); +CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1); +CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2); +CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); +CREATE INDEX log_data_index ON eth.log_cids USING btree (log_data); -- +goose Down -- log indexes diff --git a/db/migrations/00015_create_postgraphile_triggers.sql b/db/migrations/00016_create_postgraphile_triggers.sql similarity index 100% rename from db/migrations/00015_create_postgraphile_triggers.sql rename to db/migrations/00016_create_postgraphile_triggers.sql diff --git a/db/migrations/00016_create_db_version_table.sql b/db/migrations/00017_create_db_version_table.sql similarity index 100% rename from db/migrations/00016_create_db_version_table.sql rename to db/migrations/00017_create_db_version_table.sql diff --git a/db/migrations/00017_create_eth_meta_schema.sql b/db/migrations/00018_create_eth_meta_schema.sql similarity index 100% rename from db/migrations/00017_create_eth_meta_schema.sql rename to db/migrations/00018_create_eth_meta_schema.sql diff --git a/db/migrations/00018_create_watched_addresses_table.sql b/db/migrations/00019_create_watched_addresses_table.sql similarity index 100% rename from db/migrations/00018_create_watched_addresses_table.sql rename to db/migrations/00019_create_watched_addresses_table.sql diff --git a/db/migrations/00019_update_db_version.sql b/db/migrations/00019_update_db_version.sql deleted file mode 100644 index 7bf47f0..0000000 --- a/db/migrations/00019_update_db_version.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -INSERT INTO public.db_version (singleton, version) VALUES (true, 'v4.0.0') -ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v4.0.0', NOW()); - --- +goose Down -DELETE FROM public.db_version WHERE version = 'v4.0.0'; diff --git a/db/migrations/00022_create_probes_meta_tables.sql b/db/migrations/00022_create_probes_meta_tables.sql new file mode 100644 index 0000000..f35c0b3 --- /dev/null +++ b/db/migrations/00022_create_probes_meta_tables.sql @@ -0,0 +1,97 @@ +-- +goose Up +-- peer tx represents a tx that has been seen by a peer +-- the same tx (hash) can be seen by different peers +-- or received by different probes +-- so the primary key is a composite on (raw_peer_id, tx_hash, received_by_probe) +-- this table is persistent, and continues to map probe/peer meta_data to transaction hashes +-- whether they are in the canonical tx table or the pending tx table +CREATE TABLE eth_meta.peer_tx ( + raw_peer_id bytea NOT NULL, + tx_hash VARCHAR(66) NOT NULL, + received timestamp with time zone NOT NULL, + received_by_probe integer NOT NULL +); + +CREATE TABLE eth_meta.asn ( + id BIGINT NOT NULL, + asn INTEGER NOT NULL, + registry TEXT NOT NULL, + country_code TEXT NOT NULL, + name TEXT NOT NULL +); + +CREATE TABLE eth_meta.peer ( + asn_id BIGINT NOT NULL, + prefix CIDR NOT NULL, + rdns TEXT, + raw_dht_peer_id BIGINT, + city TEXT, + country TEXT, + coords JSONB +); + +CREATE TABLE eth_meta.peer_dht ( + dht_peer_id BIGINT NOT NULL, + neighbor_id BIGINT NOT NULL, + seen TIMESTAMP WITH TIME ZONE NOT NULL, + seen_by_probe INTEGER NOT NULL +); + +CREATE TABLE eth_meta.peer_seen ( + raw_peer_id BYTEA NOT NULL, + first_seen TIMESTAMP WITH TIME ZONE NOT NULL, + probe_id INTEGER NOT NULL +); + +CREATE TABLE eth_meta.probe ( + id INTEGER NOT NULL, + ip INET NOT NULL, + deployed TIMESTAMP WITH TIME ZONE NOT NULL +); + +CREATE TABLE eth_meta.raw_dht_peer ( + id BIGINT NOT NULL, + pubkey BYTEA NOT NULL, + ip INET NOT NULL, + port INTEGER NOT NULL, + client_id TEXT, + network_id BYTEA, + genesis_hash BYTEA, + forks JSONB, + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL +); + +CREATE TABLE eth_meta.raw_peer ( + id BYTEA NOT NULL, + ip INET NOT NULL, + port INTEGER NOT NULL, + client_id TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL +); + +CREATE TABLE eth_meta.site ( + id INTEGER NOT NULL, + provider TEXT NOT NULL, + az TEXT NOT NULL, + probe_id INTEGER NOT NULL, + privkey BYTEA NOT NULL +); + +CREATE TABLE eth_meta.tx_chain ( + id BYTEA NOT NULL, + height INTEGER NOT NULL, + ts TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- +goose Down +DROP TABLE eth_meta.tx_chain; +DROP TABLE eth_meta.site; +DROP TABLE eth_meta.raw_peer; +DROP TABLE eth_meta.raw_dht_peer; +DROP TABLE eth_meta.probe; +DROP TABLE eth_meta.peer_seen; +DROP TABLE eth_meta.peer_dht; +DROP TABLE eth_meta.peer; +DROP TABLE eth_meta.asn; +DROP TABLE eth_meta.peer_tx; diff --git a/db/migrations/00023_update_db_version.sql b/db/migrations/00023_update_db_version.sql new file mode 100644 index 0000000..918822e --- /dev/null +++ b/db/migrations/00023_update_db_version.sql @@ -0,0 +1,6 @@ +-- +goose Up +INSERT INTO public.db_version (singleton, version) VALUES (true, 'v5.0.0') +ON CONFLICT (singleton) DO UPDATE SET (version, tstamp) = ('v5.0.0', NOW()); + +-- +goose Down +DELETE FROM public.db_version WHERE version = 'v5.0.0'; From a9755c6ecc9b59056a1cb06b747b6da42a86e4e3 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 30 Aug 2022 12:06:01 -0500 Subject: [PATCH 6/6] update schema --- .../00014_create_pending_tx_table.sql | 6 +- schema.sql | 185 ++++++++++++++++-- 2 files changed, 171 insertions(+), 20 deletions(-) diff --git a/db/migrations/00014_create_pending_tx_table.sql b/db/migrations/00014_create_pending_tx_table.sql index 1a8c7b3..327925a 100644 --- a/db/migrations/00014_create_pending_tx_table.sql +++ b/db/migrations/00014_create_pending_tx_table.sql @@ -1,7 +1,11 @@ -- +goose Up +-- pending tx isn't tightly associated with a block height, so we can't insert the RLP encoded tx as an IPLD block +-- in public.blocks since it is denormalized by block number (we could do something hacky like using head height +-- when the block was seen, or 0 or -1 or something) +-- instead, what we are doing for the time being is embedding the RLP here CREATE TABLE IF NOT EXISTS eth.pending_txs ( tx_hash VARCHAR(66) NOT NULL PRIMARY KEY, - mh_key TEXT NOT NULL + raw BYTEA NOT NULL ); -- +goose Down diff --git a/schema.sql b/schema.sql index 7ef2304..8f9f0f7 100644 --- a/schema.sql +++ b/schema.sql @@ -439,6 +439,16 @@ CREATE TABLE eth.log_cids ( ); +-- +-- Name: pending_txs; Type: TABLE; Schema: eth; Owner: - +-- + +CREATE TABLE eth.pending_txs ( + tx_hash character varying(66) NOT NULL, + raw bytea NOT NULL +); + + -- -- Name: receipt_cids; Type: TABLE; Schema: eth; Owner: - -- @@ -546,6 +556,135 @@ CREATE TABLE eth.uncle_cids ( ); +-- +-- Name: asn; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.asn ( + id bigint NOT NULL, + asn integer NOT NULL, + registry text NOT NULL, + country_code text NOT NULL, + name text NOT NULL +); + + +-- +-- Name: peer; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.peer ( + asn_id bigint NOT NULL, + prefix cidr NOT NULL, + rdns text, + raw_dht_peer_id bigint, + city text, + country text, + coords jsonb +); + + +-- +-- Name: peer_dht; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.peer_dht ( + dht_peer_id bigint NOT NULL, + neighbor_id bigint NOT NULL, + seen timestamp with time zone NOT NULL, + seen_by_probe integer NOT NULL +); + + +-- +-- Name: peer_seen; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.peer_seen ( + raw_peer_id bytea NOT NULL, + first_seen timestamp with time zone NOT NULL, + probe_id integer NOT NULL +); + + +-- +-- Name: peer_tx; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.peer_tx ( + raw_peer_id bytea NOT NULL, + tx_hash character varying(66) NOT NULL, + received timestamp with time zone NOT NULL, + received_by_probe integer NOT NULL +); + + +-- +-- Name: probe; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.probe ( + id integer NOT NULL, + ip inet NOT NULL, + deployed timestamp with time zone NOT NULL +); + + +-- +-- Name: raw_dht_peer; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.raw_dht_peer ( + id bigint NOT NULL, + pubkey bytea NOT NULL, + ip inet NOT NULL, + port integer NOT NULL, + client_id text, + network_id bytea, + genesis_hash bytea, + forks jsonb, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: raw_peer; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.raw_peer ( + id bytea NOT NULL, + ip inet NOT NULL, + port integer NOT NULL, + client_id text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +-- +-- Name: site; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.site ( + id integer NOT NULL, + provider text NOT NULL, + az text NOT NULL, + probe_id integer NOT NULL, + privkey bytea NOT NULL +); + + +-- +-- Name: tx_chain; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.tx_chain ( + id bytea NOT NULL, + height integer NOT NULL, + ts timestamp with time zone NOT NULL +); + + -- -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - -- @@ -671,6 +810,14 @@ ALTER TABLE ONLY eth.log_cids ADD CONSTRAINT log_cids_pkey PRIMARY KEY (rct_id, index, header_id, block_number); +-- +-- Name: pending_txs pending_txs_pkey; Type: CONSTRAINT; Schema: eth; Owner: - +-- + +ALTER TABLE ONLY eth.pending_txs + ADD CONSTRAINT pending_txs_pkey PRIMARY KEY (tx_hash); + + -- -- Name: receipt_cids receipt_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: - -- @@ -770,7 +917,7 @@ CREATE INDEX access_list_block_number_index ON eth.access_list_elements USING br -- Name: access_list_element_address_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING hash (address); +CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING btree (address); -- @@ -791,14 +938,14 @@ CREATE INDEX account_block_number_index ON eth.state_accounts USING brin (block_ -- Name: account_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX account_header_id_index ON eth.state_accounts USING hash (header_id); +CREATE INDEX account_header_id_index ON eth.state_accounts USING btree (header_id); -- -- Name: account_storage_root_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX account_storage_root_index ON eth.state_accounts USING hash (storage_root); +CREATE INDEX account_storage_root_index ON eth.state_accounts USING btree (storage_root); -- @@ -826,7 +973,7 @@ CREATE UNIQUE INDEX header_mh_block_number_index ON eth.header_cids USING btree -- Name: log_address_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_address_index ON eth.log_cids USING hash (address); +CREATE INDEX log_address_index ON eth.log_cids USING btree (address); -- @@ -847,14 +994,14 @@ CREATE INDEX log_cid_index ON eth.log_cids USING btree (leaf_cid); -- Name: log_data_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_data_index ON eth.log_cids USING hash (log_data); +CREATE INDEX log_data_index ON eth.log_cids USING btree (log_data); -- -- Name: log_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_header_id_index ON eth.log_cids USING hash (header_id); +CREATE INDEX log_header_id_index ON eth.log_cids USING btree (header_id); -- @@ -868,28 +1015,28 @@ CREATE INDEX log_leaf_mh_block_number_index ON eth.log_cids USING btree (leaf_mh -- Name: log_topic0_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic0_index ON eth.log_cids USING hash (topic0); +CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0); -- -- Name: log_topic1_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic1_index ON eth.log_cids USING hash (topic1); +CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1); -- -- Name: log_topic2_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic2_index ON eth.log_cids USING hash (topic2); +CREATE INDEX log_topic2_index ON eth.log_cids USING btree (topic2); -- -- Name: log_topic3_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX log_topic3_index ON eth.log_cids USING hash (topic3); +CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); -- @@ -910,14 +1057,14 @@ CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_h -- Name: rct_contract_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_contract_index ON eth.receipt_cids USING hash (contract); +CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); -- -- Name: rct_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_header_id_index ON eth.receipt_cids USING hash (header_id); +CREATE INDEX rct_header_id_index ON eth.receipt_cids USING btree (header_id); -- @@ -952,7 +1099,7 @@ CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid); -- Name: state_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX state_header_id_index ON eth.state_cids USING hash (header_id); +CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id); -- @@ -980,7 +1127,7 @@ CREATE INDEX state_node_type_index ON eth.state_cids USING btree (node_type); -- Name: state_root_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX state_root_index ON eth.header_cids USING hash (state_root); +CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root); -- @@ -1001,7 +1148,7 @@ CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid); -- Name: storage_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX storage_header_id_index ON eth.storage_cids USING hash (header_id); +CREATE INDEX storage_header_id_index ON eth.storage_cids USING btree (header_id); -- @@ -1057,21 +1204,21 @@ CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid, block_number -- Name: tx_data_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_data_index ON eth.transaction_cids USING hash (tx_data); +CREATE INDEX tx_data_index ON eth.transaction_cids USING btree (tx_data); -- -- Name: tx_dst_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_dst_index ON eth.transaction_cids USING hash (dst); +CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst); -- -- Name: tx_header_id_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_header_id_index ON eth.transaction_cids USING hash (header_id); +CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id); -- @@ -1085,7 +1232,7 @@ CREATE INDEX tx_mh_block_number_index ON eth.transaction_cids USING btree (mh_ke -- Name: tx_src_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX tx_src_index ON eth.transaction_cids USING hash (src); +CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); --