From a500c1a49a6aa94888edeb0722b5c3d3740d893c Mon Sep 17 00:00:00 2001 From: nabarun Date: Thu, 17 Mar 2022 19:02:05 +0530 Subject: [PATCH 1/8] Add table for watched addresses in eth_meta schema --- .../00019_create_eth_meta_schema.sql | 5 ++++ .../00020_create_watched_addresses_table.sql | 10 +++++++ schema.sql | 27 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 db/migrations/00019_create_eth_meta_schema.sql create mode 100644 db/migrations/00020_create_watched_addresses_table.sql diff --git a/db/migrations/00019_create_eth_meta_schema.sql b/db/migrations/00019_create_eth_meta_schema.sql new file mode 100644 index 0000000..448cc91 --- /dev/null +++ b/db/migrations/00019_create_eth_meta_schema.sql @@ -0,0 +1,5 @@ +-- +goose Up +CREATE SCHEMA eth_meta; + +-- +goose Down +DROP SCHEMA eth_meta; diff --git a/db/migrations/00020_create_watched_addresses_table.sql b/db/migrations/00020_create_watched_addresses_table.sql new file mode 100644 index 0000000..a8f009f --- /dev/null +++ b/db/migrations/00020_create_watched_addresses_table.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE eth_meta.watched_addresses ( + address VARCHAR(66) PRIMARY KEY, + created_at BIGINT NOT NULL, + watched_at BIGINT NOT NULL, + last_filled_at BIGINT NOT NULL DEFAULT 0 +); + +-- +goose Down +DROP TABLE eth_meta.watched_addresses; diff --git a/schema.sql b/schema.sql index 7651c6c..67c8bd9 100644 --- a/schema.sql +++ b/schema.sql @@ -23,6 +23,13 @@ SET row_security = off; CREATE SCHEMA eth; +-- +-- Name: eth_meta; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA eth_meta; + + SET default_tablespace = ''; SET default_table_access_method = heap; @@ -402,6 +409,18 @@ CREATE TABLE eth.uncle_cids ( ); +-- +-- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.watched_addresses ( + address character varying(66) NOT NULL, + created_at bigint NOT NULL, + watched_at bigint NOT NULL, + last_filled_at bigint DEFAULT 0 NOT NULL +); + + -- -- Name: blocks; Type: TABLE; Schema: public; Owner: - -- @@ -562,6 +581,14 @@ ALTER TABLE ONLY eth.uncle_cids ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash); +-- +-- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - +-- + +ALTER TABLE ONLY eth_meta.watched_addresses + ADD CONSTRAINT watched_addresses_pkey PRIMARY KEY (address); + + -- -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- From 35b40e6ff79e38286b67ea7e74759e8ea8d5ccec Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Fri, 18 Mar 2022 14:35:07 -0400 Subject: [PATCH 2/8] Add `known_gaps` table - DO NOT MERGE YET!! We will probably need to change the `modulus_block_number` columns name --- schema.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/schema.sql b/schema.sql index 7651c6c..23613f8 100644 --- a/schema.sql +++ b/schema.sql @@ -469,6 +469,18 @@ CREATE TABLE public.nodes ( ); +-- +-- Name: known_gaps; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE eth.known_gaps ( + starting_block_number bigint PRIMARY KEY, + ending_block_number bigint, + checked_out boolean, + modulus_block_number INT +); + + -- -- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: - -- From a1b75c31e99094ccd04a5f45c3eb8b2adbd2fd49 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Sat, 19 Mar 2022 09:17:48 -0400 Subject: [PATCH 3/8] Update schema.sql --- schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.sql b/schema.sql index 23613f8..34b8cf0 100644 --- a/schema.sql +++ b/schema.sql @@ -477,7 +477,7 @@ CREATE TABLE eth.known_gaps ( starting_block_number bigint PRIMARY KEY, ending_block_number bigint, checked_out boolean, - modulus_block_number INT + processing_key INT ); From 679f3e8d79fd22bc5d9fd52f32d07f7514d88bfd Mon Sep 17 00:00:00 2001 From: nabarun Date: Mon, 21 Mar 2022 17:07:46 +0530 Subject: [PATCH 4/8] Remove GO111MODULE flag when installing goose in makefile Passing GO111MODULE=off with go get throws error. The CI fails due to this when using the makefile. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 223d7fd..02fbe49 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ BIN = $(GOPATH)/bin ## Migration tool GOOSE = $(BIN)/goose $(BIN)/goose: - GO111MODULE=off go get -u github.com/pressly/goose/cmd/goose + go get -u github.com/pressly/goose/cmd/goose .PHONY: installtools installtools: | $(GOOSE) From e99787242e5f2dd67bc9c37a81a6368ef41e8041 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Thu, 31 Mar 2022 10:45:11 -0400 Subject: [PATCH 5/8] Add Migration file --- db/migrations/00019_create_known_gaps_table.sql | 10 ++++++++++ schema.sql | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 db/migrations/00019_create_known_gaps_table.sql diff --git a/db/migrations/00019_create_known_gaps_table.sql b/db/migrations/00019_create_known_gaps_table.sql new file mode 100644 index 0000000..b6235b3 --- /dev/null +++ b/db/migrations/00019_create_known_gaps_table.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE eth.known_gaps ( + starting_block_number bigint PRIMARY KEY, + ending_block_number bigint, + checked_out boolean, + processing_key bigint +); + +-- +goose Down +DROP TABLE eth.known_gaps; \ No newline at end of file diff --git a/schema.sql b/schema.sql index 34b8cf0..cc9a041 100644 --- a/schema.sql +++ b/schema.sql @@ -477,7 +477,7 @@ CREATE TABLE eth.known_gaps ( starting_block_number bigint PRIMARY KEY, ending_block_number bigint, checked_out boolean, - processing_key INT + processing_key bigint ); From b2f8f63a65107fb9da3df13a6829672b82a00202 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Thu, 31 Mar 2022 10:47:29 -0400 Subject: [PATCH 6/8] Update file name --- ...ate_known_gaps_table.sql => 00031_create_known_gaps_table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/migrations/{00019_create_known_gaps_table.sql => 00031_create_known_gaps_table.sql} (100%) diff --git a/db/migrations/00019_create_known_gaps_table.sql b/db/migrations/00031_create_known_gaps_table.sql similarity index 100% rename from db/migrations/00019_create_known_gaps_table.sql rename to db/migrations/00031_create_known_gaps_table.sql From 2ffb98c6f7608cbc14f757dcbb924e4eda2fc91c Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Thu, 31 Mar 2022 12:06:11 -0400 Subject: [PATCH 7/8] Change file name --- ...ate_known_gaps_table.sql => 00021_create_known_gaps_table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/migrations/{00031_create_known_gaps_table.sql => 00021_create_known_gaps_table.sql} (100%) diff --git a/db/migrations/00031_create_known_gaps_table.sql b/db/migrations/00021_create_known_gaps_table.sql similarity index 100% rename from db/migrations/00031_create_known_gaps_table.sql rename to db/migrations/00021_create_known_gaps_table.sql From 201cadbd490c7a9853e224fc3370435956fa2c42 Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 31 Mar 2022 12:45:27 -0500 Subject: [PATCH 8/8] update the pre- and post- batch sets with new meta schema and tables --- .../00021_create_known_gaps_table.sql | 12 +++---- ...t2.sql => 00023_create_pk_constraints.sql} | 24 ++++++++++++++ .../00023_create_pk_constraints_part1.sql | 25 --------------- .../00024_create_eth_meta_schema.sql | 5 +++ .../00025_create_watched_addresses_table.sql | 10 ++++++ .../00026_create_known_gaps_table.sql | 10 ++++++ ...ogged.sql => 00027_make_tables_logged.sql} | 0 ...ions.sql => 00028_create_fk_relations.sql} | 0 ...=> 00029_create_postgraphile_comments.sql} | 0 ...dexes.sql => 00030_create_cid_indexes.sql} | 0 ....sql => 00031_create_stored_functions.sql} | 0 ...=> 00032_create_postgraphile_triggers.sql} | 0 schema.sql | 32 ++++++++++++------- 13 files changed, 75 insertions(+), 43 deletions(-) rename db/post_batch_processing_migrations/{00024_create_pk_constraints_part2.sql => 00023_create_pk_constraints.sql} (62%) delete mode 100644 db/post_batch_processing_migrations/00023_create_pk_constraints_part1.sql create mode 100644 db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql create mode 100644 db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql create mode 100644 db/post_batch_processing_migrations/00026_create_known_gaps_table.sql rename db/post_batch_processing_migrations/{00025_make_tables_logged.sql => 00027_make_tables_logged.sql} (100%) rename db/post_batch_processing_migrations/{00026_create_fk_relations.sql => 00028_create_fk_relations.sql} (100%) rename db/post_batch_processing_migrations/{00027_create_postgraphile_comments.sql => 00029_create_postgraphile_comments.sql} (100%) rename db/post_batch_processing_migrations/{00028_create_cid_indexes.sql => 00030_create_cid_indexes.sql} (100%) rename db/post_batch_processing_migrations/{00029_create_stored_functions.sql => 00031_create_stored_functions.sql} (100%) rename db/post_batch_processing_migrations/{00030_create_postgraphile_triggers.sql => 00032_create_postgraphile_triggers.sql} (100%) diff --git a/db/migrations/00021_create_known_gaps_table.sql b/db/migrations/00021_create_known_gaps_table.sql index b6235b3..4a7c730 100644 --- a/db/migrations/00021_create_known_gaps_table.sql +++ b/db/migrations/00021_create_known_gaps_table.sql @@ -1,10 +1,10 @@ -- +goose Up -CREATE TABLE eth.known_gaps ( - starting_block_number bigint PRIMARY KEY, - ending_block_number bigint, - checked_out boolean, - processing_key bigint +CREATE TABLE eth_meta.known_gaps ( + starting_block_number bigint PRIMARY KEY, + ending_block_number bigint, + checked_out boolean, + processing_key bigint ); -- +goose Down -DROP TABLE eth.known_gaps; \ No newline at end of file +DROP TABLE eth_meta.known_gaps; diff --git a/db/post_batch_processing_migrations/00024_create_pk_constraints_part2.sql b/db/post_batch_processing_migrations/00023_create_pk_constraints.sql similarity index 62% rename from db/post_batch_processing_migrations/00024_create_pk_constraints_part2.sql rename to db/post_batch_processing_migrations/00023_create_pk_constraints.sql index ba40f0f..1c3e2cb 100644 --- a/db/post_batch_processing_migrations/00024_create_pk_constraints_part2.sql +++ b/db/post_batch_processing_migrations/00023_create_pk_constraints.sql @@ -1,4 +1,16 @@ -- +goose Up +ALTER TABLE public.nodes +ADD CONSTRAINT pk_public_nodes PRIMARY KEY (node_id); + +ALTER TABLE eth.header_cids +ADD CONSTRAINT pk_eth_header_cids PRIMARY KEY (block_hash); + +ALTER TABLE eth.uncle_cids +ADD CONSTRAINT pk_eth_uncle_cids PRIMARY KEY (block_hash); + +ALTER TABLE eth.transaction_cids +ADD CONSTRAINT pk_eth_transaction_cids PRIMARY KEY (tx_hash); + ALTER TABLE eth.receipt_cids ADD CONSTRAINT pk_eth_receipt_cids PRIMARY KEY (tx_id); @@ -35,3 +47,15 @@ DROP CONSTRAINT pk_eth_access_list_elements; ALTER TABLE eth.receipt_cids DROP CONSTRAINT pk_eth_receipt_cids; + +ALTER TABLE eth.transaction_cids +DROP CONSTRAINT pk_eth_transaction_cids; + +ALTER TABLE eth.uncle_cids +DROP CONSTRAINT pk_eth_uncle_cids; + +ALTER TABLE eth.header_cids +DROP CONSTRAINT pk_eth_header_cids; + +ALTER TABLE public.nodes +DROP CONSTRAINT pk_public_nodes; diff --git a/db/post_batch_processing_migrations/00023_create_pk_constraints_part1.sql b/db/post_batch_processing_migrations/00023_create_pk_constraints_part1.sql deleted file mode 100644 index 700d273..0000000 --- a/db/post_batch_processing_migrations/00023_create_pk_constraints_part1.sql +++ /dev/null @@ -1,25 +0,0 @@ --- +goose Up -ALTER TABLE public.nodes -ADD CONSTRAINT pk_public_nodes PRIMARY KEY (node_id); - -ALTER TABLE eth.header_cids -ADD CONSTRAINT pk_eth_header_cids PRIMARY KEY (block_hash); - -ALTER TABLE eth.uncle_cids -ADD CONSTRAINT pk_eth_uncle_cids PRIMARY KEY (block_hash); - -ALTER TABLE eth.transaction_cids -ADD CONSTRAINT pk_eth_transaction_cids PRIMARY KEY (tx_hash); - --- +goose Down -ALTER TABLE eth.transaction_cids -DROP CONSTRAINT pk_eth_transaction_cids; - -ALTER TABLE eth.uncle_cids -DROP CONSTRAINT pk_eth_uncle_cids; - -ALTER TABLE eth.header_cids -DROP CONSTRAINT pk_eth_header_cids; - -ALTER TABLE public.nodes -DROP CONSTRAINT pk_public_nodes; diff --git a/db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql b/db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql new file mode 100644 index 0000000..448cc91 --- /dev/null +++ b/db/post_batch_processing_migrations/00024_create_eth_meta_schema.sql @@ -0,0 +1,5 @@ +-- +goose Up +CREATE SCHEMA eth_meta; + +-- +goose Down +DROP SCHEMA eth_meta; diff --git a/db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql b/db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql new file mode 100644 index 0000000..a8f009f --- /dev/null +++ b/db/post_batch_processing_migrations/00025_create_watched_addresses_table.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE eth_meta.watched_addresses ( + address VARCHAR(66) PRIMARY KEY, + created_at BIGINT NOT NULL, + watched_at BIGINT NOT NULL, + last_filled_at BIGINT NOT NULL DEFAULT 0 +); + +-- +goose Down +DROP TABLE eth_meta.watched_addresses; diff --git a/db/post_batch_processing_migrations/00026_create_known_gaps_table.sql b/db/post_batch_processing_migrations/00026_create_known_gaps_table.sql new file mode 100644 index 0000000..4f8032b --- /dev/null +++ b/db/post_batch_processing_migrations/00026_create_known_gaps_table.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE eth_meta.known_gaps ( + starting_block_number bigint PRIMARY KEY, + ending_block_number bigint, + checked_out boolean, + processing_key bigint +); + +-- +goose Down +DROP TABLE eth_meta.known_gaps; diff --git a/db/post_batch_processing_migrations/00025_make_tables_logged.sql b/db/post_batch_processing_migrations/00027_make_tables_logged.sql similarity index 100% rename from db/post_batch_processing_migrations/00025_make_tables_logged.sql rename to db/post_batch_processing_migrations/00027_make_tables_logged.sql diff --git a/db/post_batch_processing_migrations/00026_create_fk_relations.sql b/db/post_batch_processing_migrations/00028_create_fk_relations.sql similarity index 100% rename from db/post_batch_processing_migrations/00026_create_fk_relations.sql rename to db/post_batch_processing_migrations/00028_create_fk_relations.sql diff --git a/db/post_batch_processing_migrations/00027_create_postgraphile_comments.sql b/db/post_batch_processing_migrations/00029_create_postgraphile_comments.sql similarity index 100% rename from db/post_batch_processing_migrations/00027_create_postgraphile_comments.sql rename to db/post_batch_processing_migrations/00029_create_postgraphile_comments.sql diff --git a/db/post_batch_processing_migrations/00028_create_cid_indexes.sql b/db/post_batch_processing_migrations/00030_create_cid_indexes.sql similarity index 100% rename from db/post_batch_processing_migrations/00028_create_cid_indexes.sql rename to db/post_batch_processing_migrations/00030_create_cid_indexes.sql diff --git a/db/post_batch_processing_migrations/00029_create_stored_functions.sql b/db/post_batch_processing_migrations/00031_create_stored_functions.sql similarity index 100% rename from db/post_batch_processing_migrations/00029_create_stored_functions.sql rename to db/post_batch_processing_migrations/00031_create_stored_functions.sql diff --git a/db/post_batch_processing_migrations/00030_create_postgraphile_triggers.sql b/db/post_batch_processing_migrations/00032_create_postgraphile_triggers.sql similarity index 100% rename from db/post_batch_processing_migrations/00030_create_postgraphile_triggers.sql rename to db/post_batch_processing_migrations/00032_create_postgraphile_triggers.sql diff --git a/schema.sql b/schema.sql index e588c8a..2e71a7d 100644 --- a/schema.sql +++ b/schema.sql @@ -409,6 +409,18 @@ CREATE TABLE eth.uncle_cids ( ); +-- +-- Name: known_gaps; Type: TABLE; Schema: eth_meta; Owner: - +-- + +CREATE TABLE eth_meta.known_gaps ( + starting_block_number bigint NOT NULL, + ending_block_number bigint, + checked_out boolean, + processing_key bigint +); + + -- -- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: - -- @@ -488,18 +500,6 @@ CREATE TABLE public.nodes ( ); --- --- Name: known_gaps; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE eth.known_gaps ( - starting_block_number bigint PRIMARY KEY, - ending_block_number bigint, - checked_out boolean, - processing_key bigint -); - - -- -- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: - -- @@ -593,6 +593,14 @@ ALTER TABLE ONLY eth.uncle_cids ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash); +-- +-- Name: known_gaps known_gaps_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - +-- + +ALTER TABLE ONLY eth_meta.known_gaps + ADD CONSTRAINT known_gaps_pkey PRIMARY KEY (starting_block_number); + + -- -- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: - --