major refactor part 2: remove cold import, full sync, generalize node table

db/migrations/00001_create_blocks_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE public.eth_blocks (
-  id            SERIAL PRIMARY KEY,
-  difficulty    BIGINT,
-  extra_data    VARCHAR,
-  gas_limit      BIGINT,
-  gas_used       BIGINT,
-  hash          VARCHAR(66),
-  miner         VARCHAR(42),
-  nonce         VARCHAR(20),
-  "number"      BIGINT,
-  parent_hash    VARCHAR(66),
-  reward        NUMERIC,
-  uncles_reward NUMERIC,
-  "size"        VARCHAR,
-  "time"        BIGINT,
-  is_final      BOOLEAN,
-  uncle_hash    VARCHAR(66)
-);
-
-
--- +goose Down
-DROP TABLE public.eth_blocks;

db/migrations/00001_create_nodes_table.sql

@@ -3,7 +3,7 @@ CREATE TABLE nodes (
   id            SERIAL PRIMARY KEY,
   client_name   VARCHAR,
   genesis_block VARCHAR(66),
-  network_id    NUMERIC,
+  network_id    VARCHAR,
   node_id       VARCHAR(128),
   CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
 );

db/migrations/00002_create_full_sync_transactions_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_transactions (
-  id          SERIAL PRIMARY KEY,
-  block_id    INTEGER NOT NULL REFERENCES eth_blocks(id) ON DELETE CASCADE,
-  gas_limit    NUMERIC,
-  gas_price    NUMERIC,
-  hash        VARCHAR(66),
-  input_data  BYTEA,
-  nonce       NUMERIC,
-  raw         BYTEA,
-  tx_from     VARCHAR(66),
-  tx_index    INTEGER,
-  tx_to       VARCHAR(66),
-  "value"     NUMERIC
-);
-
--- +goose Down
-DROP TABLE full_sync_transactions;

db/migrations/00003_add_block_index_to_blocks.sql

@@ -1,6 +0,0 @@
--- +goose Up
-CREATE INDEX number_index ON eth_blocks (number);
-
-
--- +goose Down
-DROP INDEX number_index;

db/migrations/00003_create_headers_table.sql

@@ -7,7 +7,7 @@ CREATE TABLE public.headers
     raw                  JSONB,
     block_timestamp      NUMERIC,
     check_count          INTEGER NOT NULL DEFAULT 0,
-    eth_node_id          INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
+    node_id              INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
     eth_node_fingerprint VARCHAR(128),
     UNIQUE (block_number, hash, eth_node_fingerprint)
 );

db/migrations/00004_create_contracts_table.sql

@@ -1,10 +0,0 @@
--- +goose Up
-CREATE TABLE watched_contracts
-(
-  contract_id   SERIAL PRIMARY KEY,
-  contract_abi  json,
-  contract_hash VARCHAR(66) UNIQUE
-);
-
--- +goose Down
-DROP TABLE watched_contracts;

db/migrations/00006_add_node_fk_to_blocks.sql

@@ -1,11 +0,0 @@
--- +goose Up
-ALTER TABLE eth_blocks
-  ADD COLUMN node_id INTEGER NOT NULL,
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (node_id)
-REFERENCES nodes (id)
-ON DELETE CASCADE;
-
--- +goose Down
-ALTER TABLE eth_blocks
-  DROP COLUMN node_id;

db/migrations/00007_create_full_sync_logs_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_logs
-(
-    id           SERIAL PRIMARY KEY,
-    block_number BIGINT,
-    address      VARCHAR(66),
-    tx_hash      VARCHAR(66),
-    index        BIGINT,
-    topic0       VARCHAR(66),
-    topic1       VARCHAR(66),
-    topic2       VARCHAR(66),
-    topic3       VARCHAR(66),
-    data         TEXT
-);
-
-
--- +goose Down
-DROP TABLE full_sync_logs;

db/migrations/00008_add_node_block_number_unique_constraint_to_blocks.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE eth_blocks
-  ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
-
--- +goose Down
-ALTER TABLE eth_blocks
-  DROP CONSTRAINT node_id_block_number_uc;

db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX block_id_index ON full_sync_transactions (block_id);
-
--- +goose Down
-DROP INDEX block_id_index;

db/migrations/00010_add_node_id_index_to_blocks.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX node_id_index ON eth_blocks (node_id);
-
--- +goose Down
-DROP INDEX node_id_index;

db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX tx_to_index ON full_sync_transactions(tx_to);
-
--- +goose Down
-DROP INDEX tx_to_index;

db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX tx_from_index ON full_sync_transactions(tx_from);
-
--- +goose Down
-DROP INDEX tx_from_index;

db/migrations/00012_create_eth_header_cids_table.sql

@@ -6,6 +6,7 @@ CREATE TABLE eth.header_cids (
   parent_hash           VARCHAR(66) NOT NULL,
   cid                   TEXT NOT NULL,
   td                    BIGINT,
+  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
   UNIQUE (block_number, block_hash)
 );
 

db/migrations/00014_create_receipts_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_receipts
-(
-    id                  SERIAL PRIMARY KEY,
-    transaction_id      INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
-    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
-    cumulative_gas_used NUMERIC,
-    gas_used            NUMERIC,
-    state_root          VARCHAR(66),
-    status              INTEGER,
-    tx_hash             VARCHAR(66)
-);
-
-
--- +goose Down
-DROP TABLE full_sync_receipts;

db/migrations/00015_add_transaction_id_index_to_receipts.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE INDEX transaction_id_index ON full_sync_receipts (transaction_id);
-
--- +goose Down
-DROP INDEX transaction_id_index;

db/migrations/00016_add_receipts_fk_to_logs.sql

@@ -1,17 +0,0 @@
--- +goose Up
-ALTER TABLE full_sync_logs
-    ADD COLUMN receipt_id INT;
-
-ALTER TABLE full_sync_logs
-    ADD CONSTRAINT receipts_fk
-        FOREIGN KEY (receipt_id)
-            REFERENCES full_sync_receipts (id)
-            ON DELETE CASCADE;
-
-
--- +goose Down
-ALTER TABLE full_sync_logs
-    DROP CONSTRAINT receipts_fk;
-
-ALTER TABLE full_sync_logs
-    DROP COLUMN receipt_id;

db/migrations/00017_create_log_filters.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE log_filters (
-  id         SERIAL,
-  name       VARCHAR NOT NULL CHECK (name <> ''),
-  from_block BIGINT CHECK (from_block >= 0),
-  to_block   BIGINT CHECK (from_block >= 0),
-  address    VARCHAR(66),
-  topic0     VARCHAR(66),
-  topic1     VARCHAR(66),
-  topic2     VARCHAR(66),
-  topic3     VARCHAR(66),
-  CONSTRAINT name_uc UNIQUE (name)
-);
-
--- +goose Down
-DROP TABLE log_filters;

db/migrations/00018_create_watched_event_logs.sql

@@ -1,32 +0,0 @@
--- +goose Up
-CREATE VIEW block_stats AS
-SELECT max(block_number) AS max_block,
-       min(block_number) AS min_block
-FROM full_sync_logs;
-
-CREATE VIEW watched_event_logs AS
-SELECT log_filters.name,
-       full_sync_logs.id,
-       block_number,
-       full_sync_logs.address,
-       tx_hash,
-       index,
-       full_sync_logs.topic0,
-       full_sync_logs.topic1,
-       full_sync_logs.topic2,
-       full_sync_logs.topic3,
-       data,
-       receipt_id
-FROM log_filters
-         CROSS JOIN block_stats
-         JOIN full_sync_logs ON full_sync_logs.address = log_filters.address
-    AND full_sync_logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
-    AND full_sync_logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
-WHERE (log_filters.topic0 = full_sync_logs.topic0 OR log_filters.topic0 ISNULL)
-  AND (log_filters.topic1 = full_sync_logs.topic1 OR log_filters.topic1 ISNULL)
-  AND (log_filters.topic2 = full_sync_logs.topic2 OR log_filters.topic2 ISNULL)
-  AND (log_filters.topic3 = full_sync_logs.topic3 OR log_filters.topic3 ISNULL);
-
--- +goose Down
-DROP VIEW watched_event_logs;
-DROP VIEW block_stats;

db/migrations/00019_update_log_filters_to_block_constraint.sql

@@ -1,14 +0,0 @@
--- +goose Up
-ALTER TABLE log_filters
-  DROP CONSTRAINT log_filters_from_block_check1;
-
-ALTER TABLE log_filters
-  ADD CONSTRAINT log_filters_to_block_check CHECK (to_block >= 0);
-
-
--- +goose Down
-ALTER TABLE log_filters
-  DROP CONSTRAINT log_filters_to_block_check;
-
-ALTER TABLE log_filters
-  ADD CONSTRAINT log_filters_from_block_check1 CHECK (to_block >= 0);

db/migrations/00020_create_btc_header_cids_table.sql

@@ -7,6 +7,7 @@ CREATE TABLE btc.header_cids (
 	cid          TEXT NOT NULL,
 	timestamp    NUMERIC NOT NULL,
 	bits         BIGINT NOT NULL,
+  node_id      INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
 	UNIQUE (block_number, block_hash)
 );
 

db/migrations/00020_rename_node_table.sql

@@ -1,43 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes RENAME TO eth_nodes;
-
-ALTER TABLE public.eth_nodes RENAME COLUMN node_id TO eth_node_id;
-
-ALTER TABLE public.eth_nodes DROP CONSTRAINT node_uc;
-ALTER TABLE public.eth_nodes
-  ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
-
-ALTER TABLE public.eth_blocks RENAME COLUMN node_id TO eth_node_id;
-
-ALTER TABLE public.eth_blocks DROP CONSTRAINT node_id_block_number_uc;
-ALTER TABLE public.eth_blocks
-  ADD CONSTRAINT eth_node_id_block_number_uc UNIQUE (number, eth_node_id);
-
-ALTER TABLE public.eth_blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.eth_blocks
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (eth_node_id) REFERENCES eth_nodes (id) ON DELETE CASCADE;
-
-
--- +goose Down
-ALTER TABLE public.eth_nodes
-  RENAME TO nodes;
-
-ALTER TABLE public.nodes
-  RENAME COLUMN eth_node_id TO node_id;
-
-ALTER TABLE public.nodes
-  DROP CONSTRAINT eth_node_uc;
-ALTER TABLE public.nodes
-  ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.eth_blocks RENAME COLUMN eth_node_id TO node_id;
-
-ALTER TABLE public.eth_blocks DROP CONSTRAINT eth_node_id_block_number_uc;
-ALTER TABLE public.eth_blocks
-  ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
-
-ALTER TABLE public.eth_blocks DROP CONSTRAINT node_fk;
-ALTER TABLE public.eth_blocks
-  ADD CONSTRAINT node_fk
-FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE;

db/migrations/00021_associate_receipts_with_blocks.sql

@@ -1,44 +0,0 @@
--- +goose Up
-ALTER TABLE full_sync_receipts
-  ADD COLUMN block_id INT;
-
-UPDATE full_sync_receipts
-  SET block_id = (
-    SELECT block_id FROM full_sync_transactions WHERE full_sync_transactions.id = full_sync_receipts.transaction_id
-  );
-
-ALTER TABLE full_sync_receipts
-  ALTER COLUMN block_id SET NOT NULL;
-
-ALTER TABLE full_sync_receipts
-  ADD CONSTRAINT eth_blocks_fk
-FOREIGN KEY (block_id)
-REFERENCES eth_blocks (id)
-ON DELETE CASCADE;
-
-ALTER TABLE full_sync_receipts
-  DROP COLUMN transaction_id;
-
-
--- +goose Down
-ALTER TABLE full_sync_receipts
-  ADD COLUMN transaction_id INT;
-
-CREATE INDEX transaction_id_index ON full_sync_receipts (transaction_id);
-
-UPDATE full_sync_receipts
-  SET transaction_id = (
-    SELECT id FROM full_sync_transactions WHERE full_sync_transactions.hash = full_sync_receipts.tx_hash
-  );
-
-ALTER TABLE full_sync_receipts
-  ALTER COLUMN transaction_id SET NOT NULL;
-
-ALTER TABLE full_sync_receipts
-  ADD CONSTRAINT transaction_fk
-FOREIGN KEY (transaction_id)
-REFERENCES full_sync_transactions (id)
-ON DELETE CASCADE;
-
-ALTER TABLE full_sync_receipts
-  DROP COLUMN block_id;

db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql

@@ -1,16 +0,0 @@
--- +goose Up
-ALTER TABLE eth_blocks
-  ADD COLUMN eth_node_fingerprint VARCHAR(128);
-
-UPDATE eth_blocks
-  SET eth_node_fingerprint = (
-    SELECT eth_node_id FROM eth_nodes WHERE eth_nodes.id = eth_blocks.eth_node_id
-  );
-
-ALTER TABLE eth_blocks
-  ALTER COLUMN eth_node_fingerprint SET NOT NULL;
-
-
--- +goose Down
-ALTER TABLE eth_blocks
-  DROP COLUMN eth_node_fingerprint;

db/migrations/00029_create_uncles_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE public.uncles (
-  id                    SERIAL PRIMARY KEY,
-  hash                  VARCHAR(66) NOT NULL,
-  block_id              INTEGER NOT NULL REFERENCES eth_blocks (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  miner                 VARCHAR(42) NOT NULL,
-  raw                   JSONB,
-  block_timestamp       NUMERIC,
-  eth_node_id           INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
-  eth_node_fingerprint  VARCHAR(128),
-  UNIQUE (block_id, hash)
-);
-
--- +goose Down
-DROP TABLE public.uncles;