Write event logs to database before transforming

- enables decoupling event extraction/persistence from transformation
- modifies event transformer, converter, and log chunker to accept
  payload that includes internal log database ID with log data
- remove alias for transformer pkg as shared_t
- remove unused mock watcher repository
This commit is contained in:
Rob Mulholand
2019-08-28 09:13:44 -05:00
parent 66a4e20b20
commit cb819fa9a6
18 changed files with 468 additions and 170 deletions
@@ -0,0 +1,21 @@
-- +goose Up
-- SQL in this section is executed when the migration is applied.
CREATE TABLE header_sync_logs
(
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
address VARCHAR(66),
topics BYTEA[],
data BYTEA,
block_number BIGINT,
block_hash VARCHAR(66),
tx_hash VARCHAR(66),
tx_index INTEGER,
log_index INTEGER,
raw JSONB,
UNIQUE (header_id, tx_index, log_index)
);
-- +goose Down
-- SQL in this section is executed when the migration is rolled back.
DROP TABLE header_sync_logs;
+70
View File
@@ -296,6 +296,45 @@ CREATE SEQUENCE public.goose_db_version_id_seq
ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
--
-- Name: header_sync_logs; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.header_sync_logs (
id integer NOT NULL,
header_id integer NOT NULL,
address character varying(66),
topics bytea[],
data bytea,
block_number bigint,
block_hash character varying(66),
tx_hash character varying(66),
tx_index integer,
log_index integer,
raw jsonb
);
--
-- Name: header_sync_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.header_sync_logs_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: header_sync_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.header_sync_logs_id_seq OWNED BY public.header_sync_logs.id;
--
-- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
--
@@ -650,6 +689,13 @@ ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextv
ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
--
-- Name: header_sync_logs id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_logs ALTER COLUMN id SET DEFAULT nextval('public.header_sync_logs_id_seq'::regclass);
--
-- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -787,6 +833,22 @@ ALTER TABLE ONLY public.goose_db_version
ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
--
-- Name: header_sync_logs header_sync_logs_header_id_tx_index_log_index_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_logs
ADD CONSTRAINT header_sync_logs_header_id_tx_index_log_index_key UNIQUE (header_id, tx_index, log_index);
--
-- Name: header_sync_logs header_sync_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_logs
ADD CONSTRAINT header_sync_logs_pkey PRIMARY KEY (id);
--
-- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -965,6 +1027,14 @@ ALTER TABLE ONLY public.full_sync_transactions
ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: header_sync_logs header_sync_logs_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_logs
ADD CONSTRAINT header_sync_logs_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--