Merge pull request #54 from deep-stack/ng-watched-addresses-v3

Add a table for watched addresses (v3)

Makefile

@@ -8,7 +8,7 @@ BIN = $(GOPATH)/bin
 ## Migration tool
 GOOSE = $(BIN)/goose
 $(BIN)/goose:
-	GO111MODULE=off go get -u github.com/pressly/goose/cmd/goose
+	go get -u github.com/pressly/goose/cmd/goose
 
 .PHONY: installtools
 installtools: | $(GOOSE)
@@ -57,13 +57,11 @@ rollback_to: $(GOOSE) checkmigration checkdbvars
 .PHONY: `rollback_pre_batch_set`
 rollback_pre_batch_set: $(GOOSE) checkdbvars
 	$(GOOSE) -dir db/pre_batch_processing_migrations postgres "$(CONNECT_STRING)" down
-	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 ## Rollback post_batch_set
 .PHONY: rollback_post_batch_set
 rollback_post_batch_set: $(GOOSE) checkdbvars
 	$(GOOSE) -dir db/post_batch_processing_migrations postgres "$(CONNECT_STRING)" down
-	pg_dump -O -s $(CONNECT_STRING) > schema.sql
 
 ## Apply the next up migration
 .PHONY: migrate_up_by_one

db/migrations/00014_create_cid_indexes.sql

@@ -1,8 +1,8 @@
 -- +goose Up
 -- header indexes
 CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
 
@@ -11,8 +11,8 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 -- transaction indexes
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 

db/migrations/00019_create_eth_meta_schema.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE SCHEMA eth_meta;
+
+-- +goose Down
+DROP SCHEMA eth_meta;

db/migrations/00020_create_watched_addresses_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE eth_meta.watched_addresses (
+    address                 VARCHAR(66) PRIMARY KEY,
+    created_at              BIGINT NOT NULL,
+    watched_at              BIGINT NOT NULL,
+    last_filled_at          BIGINT NOT NULL DEFAULT 0
+);
+
+-- +goose Down
+DROP TABLE eth_meta.watched_addresses;

db/post_batch_processing_migrations/00018_create_log_indexes.sql

@@ -1,15 +1,15 @@
 -- +goose Up
-CREATE INDEX log_mh_index_index ON eth.log_cids USING btree (leaf_mh_key);
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
 CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_hash_index ON eth.header_cids USING btree (block_hash);
+CREATE INDEX header_block_hash_index ON eth.header_cids USING btree (block_hash);
-CREATE INDEX tx_header_hash_index ON eth.transaction_cids USING btree (header_id);
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX transaction_hash_index ON eth.transaction_cids USING btree (tx_hash);
+CREATE INDEX tx_tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
-CREATE INDEX log_tx_hash_index ON eth.log_cids USING btree (rct_id);
+CREATE INDEX log_cids_index ON eth.log_cids USING btree (rct_id, index);
 
 -- +goose Down
-DROP INDEX eth.log_tx_hash_index;
+DROP INDEX eth.log_cids_index;
-DROP INDEX eth.transaction_hash_index;
+DROP INDEX eth.tx_tx_hash_index;
-DROP INDEX eth.tx_header_hash_index;
+DROP INDEX eth.tx_header_id_index;
-DROP INDEX eth.header_hash_index;
+DROP INDEX eth.header_block_hash_index;
 DROP INDEX eth.block_number_index;
-DROP INDEX eth.log_mh_index_index;
+DROP INDEX eth.log_mh_index;

db/post_batch_processing_migrations/00019_create_log_mhk_fk.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+-- +goose Down
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_leaf_mh_key;

db/post_batch_processing_migrations/00019_remove_log_indexes.sql

@@ -1,15 +0,0 @@
--- +goose Up
-DROP INDEX eth.log_tx_hash_index;
-DROP INDEX eth.transaction_hash_index;
-DROP INDEX eth.tx_header_hash_index;
-DROP INDEX eth.header_hash_index;
-DROP INDEX eth.block_number_index;
-DROP INDEX eth.log_mh_index_index;
-
--- +goose Down
-CREATE INDEX log_mh_index_index ON eth.log_cids USING btree (leaf_mh_key);
-CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_hash_index ON eth.header_cids USING btree (block_hash);
-CREATE INDEX tx_header_hash_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX transaction_hash_index ON eth.transaction_cids USING btree (tx_hash);
-CREATE INDEX log_tx_hash_index ON eth.log_cids USING btree (rct_id);

db/post_batch_processing_migrations/00020_disable_log_indexes_and_drop_mhk_fk.sql

@@ -0,0 +1,53 @@
+-- +goose Up
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT fk_log_leaf_mh_key;
+
+-- +goose Down
+ALTER TABLE eth.log_cids
+ADD CONSTRAINT fk_log_leaf_mh_key
+    FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks (key)
+    ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);

db/post_batch_processing_migrations/00021_reenable_and_drop_log_indexes.sql

@@ -0,0 +1,53 @@
+-- +goose Up
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=true
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+
+DROP INDEX eth.tx_header_id_index;
+DROP INDEX eth.block_number_index;
+DROP INDEX eth.log_mh_index;
+
+-- +goose Down
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.log_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.transaction_cids'
+);
+UPDATE pg_index
+SET indisready=false
+WHERE indrelid = (
+    SELECT oid
+    FROM pg_class
+    WHERE relname='eth.header_cids'
+);

db/post_batch_processing_migrations/00022_drop_non_unique_indexes.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+DROP INDEX eth.log_cids_index;
+DROP INDEX eth.tx_tx_hash_index;
+DROP INDEX eth.header_block_hash_index;
+
+-- +goose Down
+CREATE INDEX header_block_hash_index ON eth.header_cids USING btree (block_hash);
+CREATE INDEX tx_tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
+CREATE INDEX log_cids_index ON eth.log_cids USING btree (rct_id, index);

db/post_batch_processing_migrations/00023_create_pk_constraints_part1.sql

@@ -0,0 +1,25 @@
+-- +goose Up
+ALTER TABLE public.nodes
+ADD CONSTRAINT pk_public_nodes PRIMARY KEY (node_id);
+
+ALTER TABLE eth.header_cids
+ADD CONSTRAINT pk_eth_header_cids PRIMARY KEY (block_hash);
+
+ALTER TABLE eth.uncle_cids
+ADD CONSTRAINT pk_eth_uncle_cids PRIMARY KEY (block_hash);
+
+ALTER TABLE eth.transaction_cids
+ADD CONSTRAINT pk_eth_transaction_cids PRIMARY KEY (tx_hash);
+
+-- +goose Down
+ALTER TABLE eth.transaction_cids
+DROP CONSTRAINT pk_eth_transaction_cids;
+
+ALTER TABLE eth.uncle_cids
+DROP CONSTRAINT pk_eth_uncle_cids;
+
+ALTER TABLE eth.header_cids
+DROP CONSTRAINT pk_eth_header_cids;
+
+ALTER TABLE public.nodes
+DROP CONSTRAINT pk_public_nodes;

db/post_batch_processing_migrations/00024_create_pk_constraints_part2.sql

@@ -1,16 +1,4 @@
 -- +goose Up
-ALTER TABLE public.nodes
-ADD CONSTRAINT pk_public_nodes PRIMARY KEY (node_id);
-
-ALTER TABLE eth.header_cids
-ADD CONSTRAINT pk_eth_header_cids PRIMARY KEY (block_hash);
-
-ALTER TABLE eth.uncle_cids
-ADD CONSTRAINT pk_eth_uncle_cids PRIMARY KEY (block_hash);
-
-ALTER TABLE eth.transaction_cids
-ADD CONSTRAINT pk_eth_transaction_cids PRIMARY KEY (tx_hash);
-
 ALTER TABLE eth.receipt_cids
 ADD CONSTRAINT pk_eth_receipt_cids PRIMARY KEY (tx_id);
 
@@ -30,32 +18,20 @@ ALTER TABLE eth.state_accounts
 ADD CONSTRAINT pk_eth_state_accounts PRIMARY KEY (header_id, state_path);
 
 -- +goose Down
-ALTER TABLE public.nodes
+ALTER TABLE eth.state_accounts
-DROP CONSTRAINT pk_public_nodes;
+DROP CONSTRAINT pk_eth_state_accounts;
-
-ALTER TABLE eth.header_cids
-DROP CONSTRAINT pk_eth_header_cids;
-
-ALTER TABLE eth.uncle_cids
-DROP CONSTRAINT pk_eth_uncle_cids;
-
-ALTER TABLE eth.transaction_cids
-DROP CONSTRAINT pk_eth_transaction_cids;
-
-ALTER TABLE eth.receipt_cids
-DROP CONSTRAINT pk_eth_receipt_cids;
-
-ALTER TABLE eth.access_list_elements
-DROP CONSTRAINT pk_eth_access_list_elements;
-
-ALTER TABLE eth.log_cids
-DROP CONSTRAINT pk_eth_log_cids;
-
-ALTER TABLE eth.state_cids
-DROP CONSTRAINT pk_eth_state_cids;
 
 ALTER TABLE eth.storage_cids
 DROP CONSTRAINT pk_eth_storage_cids;
 
-ALTER TABLE eth.state_accounts
+ALTER TABLE eth.state_cids
-DROP CONSTRAINT pk_eth_state_accounts;
+DROP CONSTRAINT pk_eth_state_cids;
+
+ALTER TABLE eth.log_cids
+DROP CONSTRAINT pk_eth_log_cids;
+
+ALTER TABLE eth.access_list_elements
+DROP CONSTRAINT pk_eth_access_list_elements;
+
+ALTER TABLE eth.receipt_cids
+DROP CONSTRAINT pk_eth_receipt_cids;

db/post_batch_processing_migrations/00028_create_cid_indexes.sql

@@ -1,8 +1,8 @@
 -- +goose Up
 -- header indexes
 CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
 CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
 
@@ -11,8 +11,8 @@ CREATE INDEX uncle_header_id_index ON eth.uncle_cids USING btree (header_id);
 
 -- transaction indexes
 CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
 CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 
@@ -46,8 +46,8 @@ CREATE INDEX access_list_element_address_index ON eth.access_list_elements USING
 CREATE INDEX access_list_storage_keys_index ON eth.access_list_elements USING gin (storage_keys);
 
 -- log indexes
-CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
 CREATE INDEX log_cid_index ON  eth.log_cids USING btree (leaf_cid);
+CREATE INDEX log_mh_index ON eth.log_cids USING btree (leaf_mh_key);
 CREATE INDEX log_address_index ON eth.log_cids USING btree (address);
 CREATE INDEX log_topic0_index ON eth.log_cids USING btree (topic0);
 CREATE INDEX log_topic1_index ON eth.log_cids USING btree (topic1);
@@ -61,8 +61,8 @@ DROP INDEX eth.log_topic2_index;
 DROP INDEX eth.log_topic1_index;
 DROP INDEX eth.log_topic0_index;
 DROP INDEX eth.log_address_index;
-DROP INDEX eth.log_cid_index;
 DROP INDEX eth.log_mh_index;
+DROP INDEX eth.log_cid_index;
 
 -- access list indexes
 DROP INDEX eth.access_list_storage_keys_index;

schema.sql

@@ -23,6 +23,13 @@ SET row_security = off;
 CREATE SCHEMA eth;
 
 
+--
+-- Name: eth_meta; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA eth_meta;
+
+
 SET default_tablespace = '';
 
 SET default_table_access_method = heap;
@@ -402,6 +409,18 @@ CREATE TABLE eth.uncle_cids (
 );
 
 
+--
+-- Name: watched_addresses; Type: TABLE; Schema: eth_meta; Owner: -
+--
+
+CREATE TABLE eth_meta.watched_addresses (
+    address character varying(66) NOT NULL,
+    created_at bigint NOT NULL,
+    watched_at bigint NOT NULL,
+    last_filled_at bigint DEFAULT 0 NOT NULL
+);
+
+
 --
 -- Name: blocks; Type: TABLE; Schema: public; Owner: -
 --
@@ -562,6 +581,14 @@ ALTER TABLE ONLY eth.uncle_cids
     ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (block_hash);
 
 
+--
+-- Name: watched_addresses watched_addresses_pkey; Type: CONSTRAINT; Schema: eth_meta; Owner: -
+--
+
+ALTER TABLE ONLY eth_meta.watched_addresses
+    ADD CONSTRAINT watched_addresses_pkey PRIMARY KEY (address);
+
+
 --
 -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -626,14 +653,14 @@ CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
 -- Name: header_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+CREATE UNIQUE INDEX header_cid_index ON eth.header_cids USING btree (cid);
 
 
 --
 -- Name: header_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+CREATE UNIQUE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
 
 
 --
@@ -815,7 +842,7 @@ CREATE INDEX timestamp_index ON eth.header_cids USING brin ("timestamp");
 -- Name: tx_cid_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+CREATE UNIQUE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
 
 
 --
@@ -836,7 +863,7 @@ CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
 -- Name: tx_mh_index; Type: INDEX; Schema: eth; Owner: -
 --
 
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+CREATE UNIQUE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
 
 
 --