cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

Decouple log extraction from transformer delegation

Browse Source 
- limit missing headers results set to 100 so that extraction doesn't
  excessively block delegation
- wrap checked headers functions in repository struct
- move storage repository to factory, to correspond with event
  repository path
- remove unused files
- reformat sql
- remove line breaks in imports
This commit is contained in:
Rob Mulholand
2019-08-28 09:25:13 -05:00
parent cb819fa9a6
commit d496dad33c
81 changed files with 2049 additions and 1328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/migrations/00001_create_blocks_table.sql
+18 -17
View File
@@ -1,21 +1,22 @@
-- +goose Up
CREATE TABLE public.blocks (
id SERIAL PRIMARY KEY,
difficulty BIGINT,
extra_data VARCHAR,
gas_limit BIGINT,
gas_used BIGINT,
hash VARCHAR(66),
miner VARCHAR(42),
nonce VARCHAR(20),
"number" BIGINT,
parent_hash VARCHAR(66),
reward NUMERIC,
uncles_reward NUMERIC,
"size" VARCHAR,
"time" BIGINT,
is_final BOOLEAN,
uncle_hash VARCHAR(66)
CREATE TABLE public.blocks
(
id SERIAL PRIMARY KEY,
difficulty BIGINT,
extra_data VARCHAR,
gas_limit BIGINT,
gas_used BIGINT,
hash VARCHAR(66),
miner VARCHAR(42),
nonce VARCHAR(20),
"number" BIGINT,
parent_hash VARCHAR(66),
reward NUMERIC,
uncles_reward NUMERIC,
"size" VARCHAR,
"time" BIGINT,
is_final BOOLEAN,
uncle_hash VARCHAR(66)
);
db/migrations/00002_create_full_sync_transactions_table.sql
+14 -13
View File
@@ -1,17 +1,18 @@
-- +goose Up
CREATE TABLE full_sync_transactions (
id SERIAL PRIMARY KEY,
block_id INTEGER NOT NULL REFERENCES blocks(id) ON DELETE CASCADE,
gas_limit NUMERIC,
gas_price NUMERIC,
hash VARCHAR(66),
input_data BYTEA,
nonce NUMERIC,
raw BYTEA,
tx_from VARCHAR(66),
tx_index INTEGER,
tx_to VARCHAR(66),
"value" NUMERIC
CREATE TABLE full_sync_transactions
(
id SERIAL PRIMARY KEY,
block_id INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
gas_limit NUMERIC,
gas_price NUMERIC,
hash VARCHAR(66),
input_data BYTEA,
nonce NUMERIC,
raw BYTEA,
tx_from VARCHAR(66),
tx_index INTEGER,
tx_to VARCHAR(66),
"value" NUMERIC
);
-- +goose Down
db/migrations/00004_create_contracts_table.sql
+3 -3
View File
@@ -1,9 +1,9 @@
-- +goose Up
CREATE TABLE watched_contracts
(
contract_id SERIAL PRIMARY KEY,
contract_abi json,
contract_hash VARCHAR(66) UNIQUE
contract_id SERIAL PRIMARY KEY,
contract_abi json,
contract_hash VARCHAR(66) UNIQUE
);
-- +goose Down
db/migrations/00005_create_nodes_table.sql
+8 -7
View File
@@ -1,11 +1,12 @@
-- +goose Up
CREATE TABLE nodes (
id SERIAL PRIMARY KEY,
client_name VARCHAR,
genesis_block VARCHAR(66),
network_id NUMERIC,
node_id VARCHAR(128),
CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
CREATE TABLE nodes
(
id SERIAL PRIMARY KEY,
client_name VARCHAR,
genesis_block VARCHAR(66),
network_id NUMERIC,
node_id VARCHAR(128),
CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
);
-- +goose Down
db/migrations/00006_add_node_fk_to_blocks.sql
+6 -6
View File
@@ -1,11 +1,11 @@
-- +goose Up
ALTER TABLE blocks
ADD COLUMN node_id INTEGER NOT NULL,
ADD CONSTRAINT node_fk
FOREIGN KEY (node_id)
REFERENCES nodes (id)
ON DELETE CASCADE;
ADD COLUMN node_id INTEGER NOT NULL,
ADD CONSTRAINT node_fk
FOREIGN KEY (node_id)
REFERENCES nodes (id)
ON DELETE CASCADE;
-- +goose Down
ALTER TABLE blocks
DROP COLUMN node_id;
DROP COLUMN node_id;
db/migrations/00007_create_full_sync_logs_table.sql
+13 -13
View File
@@ -1,19 +1,19 @@
-- +goose Up
CREATE TABLE full_sync_logs (
id SERIAL PRIMARY KEY,
block_number BIGINT,
address VARCHAR(66),
tx_hash VARCHAR(66),
index BIGINT,
topic0 VARCHAR(66),
topic1 VARCHAR(66),
topic2 VARCHAR(66),
topic3 VARCHAR(66),
data TEXT,
CONSTRAINT full_sync_log_uc UNIQUE (block_number, index)
CREATE TABLE full_sync_logs
(
id SERIAL PRIMARY KEY,
block_number BIGINT,
address VARCHAR(66),
tx_hash VARCHAR(66),
index BIGINT,
topic0 VARCHAR(66),
topic1 VARCHAR(66),
topic2 VARCHAR(66),
topic3 VARCHAR(66),
data TEXT,
CONSTRAINT full_sync_log_uc UNIQUE (block_number, index)
);
-- +goose Down
DROP TABLE full_sync_logs;
db/migrations/00008_add_node_block_number_unique_constraint_to_blocks.sql
+2 -2
View File
@@ -1,7 +1,7 @@
-- +goose Up
ALTER TABLE blocks
ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
-- +goose Down
ALTER TABLE blocks
DROP CONSTRAINT node_id_block_number_uc;
DROP CONSTRAINT node_id_block_number_uc;
db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql
+1 -1
View File
@@ -1,5 +1,5 @@
-- +goose Up
CREATE INDEX tx_to_index ON full_sync_transactions(tx_to);
CREATE INDEX tx_to_index ON full_sync_transactions (tx_to);
-- +goose Down
DROP INDEX tx_to_index;
db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql
+1 -1
View File
@@ -1,5 +1,5 @@
-- +goose Up
CREATE INDEX tx_from_index ON full_sync_transactions(tx_from);
CREATE INDEX tx_from_index ON full_sync_transactions (tx_from);
-- +goose Down
DROP INDEX tx_from_index;
db/migrations/00016_add_receipts_fk_to_logs.sql
+8 -8
View File
@@ -3,21 +3,21 @@ ALTER TABLE full_sync_logs
DROP CONSTRAINT full_sync_log_uc;
ALTER TABLE full_sync_logs
ADD COLUMN receipt_id INT;
ADD COLUMN receipt_id INT;
ALTER TABLE full_sync_logs
ADD CONSTRAINT receipts_fk
FOREIGN KEY (receipt_id)
REFERENCES full_sync_receipts (id)
ON DELETE CASCADE;
ADD CONSTRAINT receipts_fk
FOREIGN KEY (receipt_id)
REFERENCES full_sync_receipts (id)
ON DELETE CASCADE;
-- +goose Down
ALTER TABLE full_sync_logs
DROP CONSTRAINT receipts_fk;
DROP CONSTRAINT receipts_fk;
ALTER TABLE full_sync_logs
DROP COLUMN receipt_id;
DROP COLUMN receipt_id;
ALTER TABLE full_sync_logs
ADD CONSTRAINT full_sync_log_uc UNIQUE (block_number, index);
ADD CONSTRAINT full_sync_log_uc UNIQUE (block_number, index);
db/migrations/00017_create_log_filters.sql
+12 -11
View File
@@ -1,15 +1,16 @@
-- +goose Up
CREATE TABLE log_filters (
id SERIAL,
name VARCHAR NOT NULL CHECK (name <> ''),
from_block BIGINT CHECK (from_block >= 0),
to_block BIGINT CHECK (from_block >= 0),
address VARCHAR(66),
topic0 VARCHAR(66),
topic1 VARCHAR(66),
topic2 VARCHAR(66),
topic3 VARCHAR(66),
CONSTRAINT name_uc UNIQUE (name)
CREATE TABLE log_filters
(
id SERIAL,
name VARCHAR NOT NULL CHECK (name <> ''),
from_block BIGINT CHECK (from_block >= 0),
to_block BIGINT CHECK (from_block >= 0),
address VARCHAR(66),
topic0 VARCHAR(66),
topic1 VARCHAR(66),
topic2 VARCHAR(66),
topic3 VARCHAR(66),
CONSTRAINT name_uc UNIQUE (name)
);
-- +goose Down
db/migrations/00018_create_watched_event_logs.sql
+24 -26
View File
@@ -1,33 +1,31 @@
-- +goose Up
CREATE VIEW block_stats AS
SELECT
max(block_number) AS max_block,
min(block_number) AS min_block
FROM full_sync_logs;
SELECT max(block_number) AS max_block,
min(block_number) AS min_block
FROM full_sync_logs;
CREATE VIEW watched_event_logs AS
SELECT
log_filters.name,
full_sync_logs.id,
block_number,
full_sync_logs.address,
tx_hash,
index,
full_sync_logs.topic0,
full_sync_logs.topic1,
full_sync_logs.topic2,
full_sync_logs.topic3,
data,
receipt_id
FROM log_filters
CROSS JOIN block_stats
JOIN full_sync_logs ON full_sync_logs.address = log_filters.address
AND full_sync_logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
AND full_sync_logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
WHERE (log_filters.topic0 = full_sync_logs.topic0 OR log_filters.topic0 ISNULL)
AND (log_filters.topic1 = full_sync_logs.topic1 OR log_filters.topic1 ISNULL)
AND (log_filters.topic2 = full_sync_logs.topic2 OR log_filters.topic2 ISNULL)
AND (log_filters.topic3 = full_sync_logs.topic3 OR log_filters.topic3 ISNULL);
SELECT log_filters.name,
full_sync_logs.id,
block_number,
full_sync_logs.address,
tx_hash,
index,
full_sync_logs.topic0,
full_sync_logs.topic1,
full_sync_logs.topic2,
full_sync_logs.topic3,
data,
receipt_id
FROM log_filters
CROSS JOIN block_stats
JOIN full_sync_logs ON full_sync_logs.address = log_filters.address
AND full_sync_logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
AND full_sync_logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
WHERE (log_filters.topic0 = full_sync_logs.topic0 OR log_filters.topic0 ISNULL)
AND (log_filters.topic1 = full_sync_logs.topic1 OR log_filters.topic1 ISNULL)
AND (log_filters.topic2 = full_sync_logs.topic2 OR log_filters.topic2 ISNULL)
AND (log_filters.topic3 = full_sync_logs.topic3 OR log_filters.topic3 ISNULL);
-- +goose Down
DROP VIEW watched_event_logs;
db/migrations/00019_update_log_filters_to_block_constraint.sql
+4 -4
View File
@@ -1,14 +1,14 @@
-- +goose Up
ALTER TABLE log_filters
DROP CONSTRAINT log_filters_from_block_check1;
DROP CONSTRAINT log_filters_from_block_check1;
ALTER TABLE log_filters
ADD CONSTRAINT log_filters_to_block_check CHECK (to_block >= 0);
ADD CONSTRAINT log_filters_to_block_check CHECK (to_block >= 0);
-- +goose Down
ALTER TABLE log_filters
DROP CONSTRAINT log_filters_to_block_check;
DROP CONSTRAINT log_filters_to_block_check;
ALTER TABLE log_filters
ADD CONSTRAINT log_filters_from_block_check1 CHECK (to_block >= 0);
ADD CONSTRAINT log_filters_from_block_check1 CHECK (to_block >= 0);
db/migrations/00020_rename_node_table.sql
+31 -22
View File
@@ -1,43 +1,52 @@
-- +goose Up
ALTER TABLE public.nodes RENAME TO eth_nodes;
ALTER TABLE public.nodes
RENAME TO eth_nodes;
ALTER TABLE public.eth_nodes RENAME COLUMN node_id TO eth_node_id;
ALTER TABLE public.eth_nodes DROP CONSTRAINT node_uc;
ALTER TABLE public.eth_nodes
ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
RENAME COLUMN node_id TO eth_node_id;
ALTER TABLE public.blocks RENAME COLUMN node_id TO eth_node_id;
ALTER TABLE public.eth_nodes
DROP CONSTRAINT node_uc;
ALTER TABLE public.eth_nodes
ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
ALTER TABLE public.blocks DROP CONSTRAINT node_id_block_number_uc;
ALTER TABLE public.blocks
ADD CONSTRAINT eth_node_id_block_number_uc UNIQUE (number, eth_node_id);
RENAME COLUMN node_id TO eth_node_id;
ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
ALTER TABLE public.blocks
ADD CONSTRAINT node_fk
FOREIGN KEY (eth_node_id) REFERENCES eth_nodes (id) ON DELETE CASCADE;
DROP CONSTRAINT node_id_block_number_uc;
ALTER TABLE public.blocks
ADD CONSTRAINT eth_node_id_block_number_uc UNIQUE (number, eth_node_id);
ALTER TABLE public.blocks
DROP CONSTRAINT node_fk;
ALTER TABLE public.blocks
ADD CONSTRAINT node_fk
FOREIGN KEY (eth_node_id) REFERENCES eth_nodes (id) ON DELETE CASCADE;
-- +goose Down
ALTER TABLE public.eth_nodes
RENAME TO nodes;
RENAME TO nodes;
ALTER TABLE public.nodes
RENAME COLUMN eth_node_id TO node_id;
RENAME COLUMN eth_node_id TO node_id;
ALTER TABLE public.nodes
DROP CONSTRAINT eth_node_uc;
DROP CONSTRAINT eth_node_uc;
ALTER TABLE public.nodes
ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
ALTER TABLE public.blocks RENAME COLUMN eth_node_id TO node_id;
ALTER TABLE public.blocks DROP CONSTRAINT eth_node_id_block_number_uc;
ALTER TABLE public.blocks
ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
RENAME COLUMN eth_node_id TO node_id;
ALTER TABLE public.blocks DROP CONSTRAINT node_fk;
ALTER TABLE public.blocks
ADD CONSTRAINT node_fk
FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE;
DROP CONSTRAINT eth_node_id_block_number_uc;
ALTER TABLE public.blocks
ADD CONSTRAINT node_id_block_number_uc UNIQUE (number, node_id);
ALTER TABLE public.blocks
DROP CONSTRAINT node_fk;
ALTER TABLE public.blocks
ADD CONSTRAINT node_fk
FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE;
db/migrations/00021_associate_receipts_with_blocks.sql
+18 -18
View File
@@ -1,44 +1,44 @@
-- +goose Up
ALTER TABLE full_sync_receipts
ADD COLUMN block_id INT;
ADD COLUMN block_id INT;
UPDATE full_sync_receipts
SET block_id = (
SET block_id = (
SELECT block_id FROM full_sync_transactions WHERE full_sync_transactions.id = full_sync_receipts.transaction_id
);
);
ALTER TABLE full_sync_receipts
ALTER COLUMN block_id SET NOT NULL;
ALTER COLUMN block_id SET NOT NULL;
ALTER TABLE full_sync_receipts
ADD CONSTRAINT blocks_fk
FOREIGN KEY (block_id)
REFERENCES blocks (id)
ON DELETE CASCADE;
ADD CONSTRAINT blocks_fk
FOREIGN KEY (block_id)
REFERENCES blocks (id)
ON DELETE CASCADE;
ALTER TABLE full_sync_receipts
DROP COLUMN transaction_id;
DROP COLUMN transaction_id;
-- +goose Down
ALTER TABLE full_sync_receipts
ADD COLUMN transaction_id INT;
ADD COLUMN transaction_id INT;
CREATE INDEX transaction_id_index ON full_sync_receipts (transaction_id);
UPDATE full_sync_receipts
SET transaction_id = (
SET transaction_id = (
SELECT id FROM full_sync_transactions WHERE full_sync_transactions.hash = full_sync_receipts.tx_hash
);
);
ALTER TABLE full_sync_receipts
ALTER COLUMN transaction_id SET NOT NULL;
ALTER COLUMN transaction_id SET NOT NULL;
ALTER TABLE full_sync_receipts
ADD CONSTRAINT transaction_fk
FOREIGN KEY (transaction_id)
REFERENCES full_sync_transactions (id)
ON DELETE CASCADE;
ADD CONSTRAINT transaction_fk
FOREIGN KEY (transaction_id)
REFERENCES full_sync_transactions (id)
ON DELETE CASCADE;
ALTER TABLE full_sync_receipts
DROP COLUMN block_id;
DROP COLUMN block_id;
db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql
+5 -5
View File
@@ -1,16 +1,16 @@
-- +goose Up
ALTER TABLE blocks
ADD COLUMN eth_node_fingerprint VARCHAR(128);
ADD COLUMN eth_node_fingerprint VARCHAR(128);
UPDATE blocks
SET eth_node_fingerprint = (
SET eth_node_fingerprint = (
SELECT eth_node_id FROM eth_nodes WHERE eth_nodes.id = blocks.eth_node_id
);
);
ALTER TABLE blocks
ALTER COLUMN eth_node_fingerprint SET NOT NULL;
ALTER COLUMN eth_node_fingerprint SET NOT NULL;
-- +goose Down
ALTER TABLE blocks
DROP COLUMN eth_node_fingerprint;
DROP COLUMN eth_node_fingerprint;
db/migrations/00023_create_headers_table.sql
+9 -8
View File
@@ -1,12 +1,13 @@
-- +goose Up
CREATE TABLE public.headers (
id SERIAL PRIMARY KEY,
hash VARCHAR(66),
block_number BIGINT,
raw JSONB,
block_timestamp NUMERIC,
eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
eth_node_fingerprint VARCHAR(128)
CREATE TABLE public.headers
(
id SERIAL PRIMARY KEY,
hash VARCHAR(66),
block_number BIGINT,
raw JSONB,
block_timestamp NUMERIC,
eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
eth_node_fingerprint VARCHAR(128)
);
-- Index is removed when table is
db/migrations/00024_create_checked_headers_table.sql
+5 -3
View File
@@ -1,7 +1,9 @@
-- +goose Up
CREATE TABLE public.checked_headers (
id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE
CREATE TABLE public.checked_headers
(
id SERIAL PRIMARY KEY,
header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
check_count INTEGER NOT NULL DEFAULT 1
);
-- +goose Down
db/migrations/00025_create_queued_storage.sql
+9 -8
View File
@@ -1,12 +1,13 @@
-- +goose Up
CREATE TABLE public.queued_storage (
id SERIAL PRIMARY KEY,
block_height BIGINT,
block_hash BYTEA,
contract BYTEA,
storage_key BYTEA,
storage_value BYTEA,
UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
CREATE TABLE public.queued_storage
(
id SERIAL PRIMARY KEY,
block_height BIGINT,
block_hash BYTEA,
contract BYTEA,
storage_key BYTEA,
storage_value BYTEA,
UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
);
-- +goose Down
db/migrations/00026_create_header_sync_transactions_table.sql
+15 -14
View File
@@ -1,18 +1,19 @@
-- +goose Up
CREATE TABLE header_sync_transactions (
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
hash VARCHAR(66),
gas_limit NUMERIC,
gas_price NUMERIC,
input_data BYTEA,
nonce NUMERIC,
raw BYTEA,
tx_from VARCHAR(44),
tx_index INTEGER,
tx_to VARCHAR(44),
"value" NUMERIC,
UNIQUE (header_id, hash)
CREATE TABLE header_sync_transactions
(
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
hash VARCHAR(66),
gas_limit NUMERIC,
gas_price NUMERIC,
input_data BYTEA,
nonce NUMERIC,
raw BYTEA,
tx_from VARCHAR(44),
tx_index INTEGER,
tx_to VARCHAR(44),
"value" NUMERIC,
UNIQUE (header_id, hash)
);
-- +goose Down
db/migrations/00028_create_uncles_table.sql
+12 -11
View File
@@ -1,15 +1,16 @@
-- +goose Up
CREATE TABLE public.uncles (
id SERIAL PRIMARY KEY,
hash VARCHAR(66) NOT NULL,
block_id INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
reward NUMERIC NOT NULL,
miner VARCHAR(42) NOT NULL,
raw JSONB,
block_timestamp NUMERIC,
eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
eth_node_fingerprint VARCHAR(128),
UNIQUE (block_id, hash)
CREATE TABLE public.uncles
(
id SERIAL PRIMARY KEY,
hash VARCHAR(66) NOT NULL,
block_id INTEGER NOT NULL REFERENCES blocks (id) ON DELETE CASCADE,
reward NUMERIC NOT NULL,
miner VARCHAR(42) NOT NULL,
raw JSONB,
block_timestamp NUMERIC,
eth_node_id INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
eth_node_fingerprint VARCHAR(128),
UNIQUE (block_id, hash)
);
-- +goose Down
db/migrations/00029_create_header_sync_logs_table.sql
+1
View File
@@ -13,6 +13,7 @@ CREATE TABLE header_sync_logs
tx_index INTEGER,
log_index INTEGER,
raw JSONB,
transformed BOOL NOT NULL DEFAULT FALSE,
UNIQUE (header_id, tx_index, log_index)
);
db/schema.sql
+4 -2
View File
@@ -131,7 +131,8 @@ ALTER SEQUENCE public.blocks_id_seq OWNED BY public.blocks.id;
CREATE TABLE public.checked_headers (
id integer NOT NULL,
header_id integer NOT NULL
header_id integer NOT NULL,
check_count integer DEFAULT 1 NOT NULL
);
@@ -311,7 +312,8 @@ CREATE TABLE public.header_sync_logs (
tx_hash character varying(66),
tx_index integer,
log_index integer,
raw jsonb
raw jsonb,
transformed boolean DEFAULT false NOT NULL
);