From dbdd16d3a2889d7120a4d779e08ff1033faa438d Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 20 Feb 2019 13:39:37 -0600 Subject: [PATCH] (VDB-380) Create separate table for ilks - reference ilk by foreign key every place it's used --- db/migrations/20180809214844_create_ilks.sql | 8 + db/migrations/20180809214845_create_frob.sql | 2 +- db/migrations/20180815031513_create_bite.sql | 2 +- .../20180831002535_create_pit_file.sql | 2 +- .../20180906225956_create_vat_init.sql | 2 +- .../20180910202607_create_drip_file.sql | 2 +- .../20180912015839_create_drip_drip.sql | 2 +- .../20180912171047_create_cat_file.sql | 2 +- .../20181002220302_create_vat_fold_table.sql | 2 +- .../20181004205815_create_vat_toll.sql | 2 +- .../20181004233804_create_vat_tune.sql | 2 +- .../20181008232020_create_vat_grab.sql | 2 +- .../20181011184449_create_vat_flux_table.sql | 2 +- .../20181015231509_create_vat_slip.sql | 2 +- ...129_create_pit_contract_storage_tables.sql | 4 +- ...131_create_vat_contract_storage_tables.sql | 14 +- db/schema.sql | 273 ++++++++++++++++-- pkg/transformers/bite/repository.go | 11 +- pkg/transformers/bite/repository_test.go | 6 +- .../cat_file/chop_lump/repository.go | 11 +- .../cat_file/chop_lump/repository_test.go | 10 +- pkg/transformers/drip_drip/repository.go | 11 +- pkg/transformers/drip_drip/repository_test.go | 6 +- pkg/transformers/drip_file/ilk/repository.go | 11 +- .../drip_file/ilk/repository_test.go | 6 +- pkg/transformers/frob/repository.go | 11 +- pkg/transformers/frob/repository_test.go | 6 +- pkg/transformers/integration_tests/bite.go | 5 +- .../integration_tests/cat_file.go | 7 +- .../integration_tests/drip_drip.go | 5 +- pkg/transformers/integration_tests/frob.go | 9 +- .../integration_tests/pit_file_ilk.go | 13 +- .../integration_tests/vat_flux.go | 9 +- .../integration_tests/vat_fold.go | 9 +- .../integration_tests/vat_grab.go | 9 +- .../integration_tests/vat_init.go | 9 +- .../integration_tests/vat_slip.go | 9 +- .../integration_tests/vat_tune.go | 9 +- pkg/transformers/pit_file/ilk/converter.go | 2 +- pkg/transformers/pit_file/ilk/repository.go | 11 +- .../pit_file/ilk/repository_test.go | 6 +- pkg/transformers/shared/repository.go | 24 ++ .../maker/maker_storage_repository.go | 42 ++- .../maker/maker_storage_repository_test.go | 33 ++- .../storage_diffs/maker/pit/repository.go | 47 ++- .../maker/pit/repository_test.go | 10 +- .../storage_diffs/maker/vat/repository.go | 180 ++++++++++-- .../maker/vat/repository_test.go | 30 +- pkg/transformers/test_data/pit_file.go | 4 +- pkg/transformers/vat_flux/repository.go | 11 +- pkg/transformers/vat_flux/repository_test.go | 6 +- pkg/transformers/vat_fold/repository.go | 11 +- pkg/transformers/vat_fold/repository_test.go | 6 +- pkg/transformers/vat_grab/repository.go | 11 +- pkg/transformers/vat_grab/repository_test.go | 6 +- pkg/transformers/vat_init/repository.go | 11 +- pkg/transformers/vat_init/repository_test.go | 6 +- pkg/transformers/vat_slip/repository.go | 11 +- pkg/transformers/vat_slip/repository_test.go | 6 +- pkg/transformers/vat_toll/repository.go | 11 +- pkg/transformers/vat_toll/repository_test.go | 6 +- pkg/transformers/vat_tune/repository.go | 11 +- pkg/transformers/vat_tune/repository_test.go | 6 +- test_config/test_config.go | 19 ++ 64 files changed, 885 insertions(+), 149 deletions(-) create mode 100644 db/migrations/20180809214844_create_ilks.sql diff --git a/db/migrations/20180809214844_create_ilks.sql b/db/migrations/20180809214844_create_ilks.sql new file mode 100644 index 00000000..5eec84cf --- /dev/null +++ b/db/migrations/20180809214844_create_ilks.sql @@ -0,0 +1,8 @@ +-- +goose Up +CREATE TABLE maker.ilks ( + id SERIAL PRIMARY KEY, + ilk TEXT UNIQUE +); + +-- +goose Down +DROP TABLE maker.ilks; diff --git a/db/migrations/20180809214845_create_frob.sql b/db/migrations/20180809214845_create_frob.sql index 6e1f5e10..a1762fd4 100644 --- a/db/migrations/20180809214845_create_frob.sql +++ b/db/migrations/20180809214845_create_frob.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.frob ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, dink NUMERIC, dart NUMERIC, diff --git a/db/migrations/20180815031513_create_bite.sql b/db/migrations/20180815031513_create_bite.sql index cd01c1f1..7af6f109 100644 --- a/db/migrations/20180815031513_create_bite.sql +++ b/db/migrations/20180815031513_create_bite.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.bite ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, ink NUMERIC, art NUMERIC, diff --git a/db/migrations/20180831002535_create_pit_file.sql b/db/migrations/20180831002535_create_pit_file.sql index 712e8792..a1be6c5d 100644 --- a/db/migrations/20180831002535_create_pit_file.sql +++ b/db/migrations/20180831002535_create_pit_file.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.pit_file_ilk ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), what TEXT, data NUMERIC, log_idx INTEGER NOT NUll, diff --git a/db/migrations/20180906225956_create_vat_init.sql b/db/migrations/20180906225956_create_vat_init.sql index 41260840..6fef3347 100644 --- a/db/migrations/20180906225956_create_vat_init.sql +++ b/db/migrations/20180906225956_create_vat_init.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_init ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), log_idx INTEGER NOT NUll, tx_idx INTEGER NOT NUll, raw_log JSONB, diff --git a/db/migrations/20180910202607_create_drip_file.sql b/db/migrations/20180910202607_create_drip_file.sql index 21716834..9255c407 100644 --- a/db/migrations/20180910202607_create_drip_file.sql +++ b/db/migrations/20180910202607_create_drip_file.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.drip_file_ilk ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), vow TEXT, tax NUMERIC, log_idx INTEGER NOT NUll, diff --git a/db/migrations/20180912015839_create_drip_drip.sql b/db/migrations/20180912015839_create_drip_drip.sql index ee951aa9..241dd088 100644 --- a/db/migrations/20180912015839_create_drip_drip.sql +++ b/db/migrations/20180912015839_create_drip_drip.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.drip_drip ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), log_idx INTEGER NOT NUll, tx_idx INTEGER NOT NUll, raw_log JSONB, diff --git a/db/migrations/20180912171047_create_cat_file.sql b/db/migrations/20180912171047_create_cat_file.sql index 6b8acd71..d47dec9b 100644 --- a/db/migrations/20180912171047_create_cat_file.sql +++ b/db/migrations/20180912171047_create_cat_file.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.cat_file_chop_lump ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), what TEXT, data NUMERIC, tx_idx INTEGER NOT NUll, diff --git a/db/migrations/20181002220302_create_vat_fold_table.sql b/db/migrations/20181002220302_create_vat_fold_table.sql index 33c54fae..bd661e21 100644 --- a/db/migrations/20181002220302_create_vat_fold_table.sql +++ b/db/migrations/20181002220302_create_vat_fold_table.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_fold ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, rate NUMERIC, log_idx INTEGER NOT NULL, diff --git a/db/migrations/20181004205815_create_vat_toll.sql b/db/migrations/20181004205815_create_vat_toll.sql index bea6df99..c838254e 100644 --- a/db/migrations/20181004205815_create_vat_toll.sql +++ b/db/migrations/20181004205815_create_vat_toll.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_toll ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, take NUMERIC, tx_idx INTEGER NOT NULL, diff --git a/db/migrations/20181004233804_create_vat_tune.sql b/db/migrations/20181004233804_create_vat_tune.sql index bd8b5d4b..60cc3b54 100644 --- a/db/migrations/20181004233804_create_vat_tune.sql +++ b/db/migrations/20181004233804_create_vat_tune.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_tune ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, v TEXT, w TEXT, diff --git a/db/migrations/20181008232020_create_vat_grab.sql b/db/migrations/20181008232020_create_vat_grab.sql index e940708f..b222aa05 100644 --- a/db/migrations/20181008232020_create_vat_grab.sql +++ b/db/migrations/20181008232020_create_vat_grab.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_grab ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, v TEXT, w TEXT, diff --git a/db/migrations/20181011184449_create_vat_flux_table.sql b/db/migrations/20181011184449_create_vat_flux_table.sql index f0e659f2..5fe48321 100644 --- a/db/migrations/20181011184449_create_vat_flux_table.sql +++ b/db/migrations/20181011184449_create_vat_flux_table.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_flux ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), src TEXT, dst TEXT, rad numeric, diff --git a/db/migrations/20181015231509_create_vat_slip.sql b/db/migrations/20181015231509_create_vat_slip.sql index 5f4a457c..3694db52 100644 --- a/db/migrations/20181015231509_create_vat_slip.sql +++ b/db/migrations/20181015231509_create_vat_slip.sql @@ -2,7 +2,7 @@ CREATE TABLE maker.vat_slip ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), guy TEXT, rad NUMERIC, tx_idx INTEGER NOT NULL, diff --git a/db/migrations/20181114200129_create_pit_contract_storage_tables.sql b/db/migrations/20181114200129_create_pit_contract_storage_tables.sql index 856b0da0..a3e54d47 100644 --- a/db/migrations/20181114200129_create_pit_contract_storage_tables.sql +++ b/db/migrations/20181114200129_create_pit_contract_storage_tables.sql @@ -10,7 +10,7 @@ CREATE TABLE maker.pit_ilk_spot ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), spot NUMERIC NOT NULL ); @@ -18,7 +18,7 @@ CREATE TABLE maker.pit_ilk_line ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), line NUMERIC NOT NULL ); diff --git a/db/migrations/20181114200131_create_vat_contract_storage_tables.sql b/db/migrations/20181114200131_create_vat_contract_storage_tables.sql index e7402896..9c7f8b0c 100644 --- a/db/migrations/20181114200131_create_vat_contract_storage_tables.sql +++ b/db/migrations/20181114200131_create_vat_contract_storage_tables.sql @@ -17,7 +17,7 @@ CREATE TABLE maker.vat_ilk_art ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), art NUMERIC NOT NULL ); @@ -25,7 +25,7 @@ CREATE TABLE maker.vat_ilk_ink ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), ink NUMERIC NOT NULL ); @@ -33,7 +33,7 @@ CREATE TABLE maker.vat_ilk_rate ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), rate NUMERIC NOT NULL ); @@ -41,7 +41,7 @@ CREATE TABLE maker.vat_ilk_take ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), take NUMERIC NOT NULL ); @@ -49,7 +49,7 @@ CREATE TABLE maker.vat_urn_art ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, art TEXT ); @@ -58,7 +58,7 @@ CREATE TABLE maker.vat_urn_ink ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), urn TEXT, ink NUMERIC NOT NULL ); @@ -67,7 +67,7 @@ CREATE TABLE maker.vat_gem ( id SERIAL PRIMARY KEY, block_number BIGINT, block_hash TEXT, - ilk TEXT, + ilk INTEGER NOT NULL REFERENCES maker.ilks (id), guy TEXT, gem NUMERIC NOT NULL ); diff --git a/db/schema.sql b/db/schema.sql index dd46db2f..14b2a8ee 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -64,7 +64,7 @@ SET default_with_oids = false; CREATE TABLE maker.bite ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, ink numeric, art numeric, @@ -104,7 +104,7 @@ ALTER SEQUENCE maker.bite_id_seq OWNED BY maker.bite.id; CREATE TABLE maker.cat_file_chop_lump ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, what text, data numeric, tx_idx integer NOT NULL, @@ -675,7 +675,7 @@ ALTER SEQUENCE maker.dent_id_seq OWNED BY maker.dent.id; CREATE TABLE maker.drip_drip ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, log_idx integer NOT NULL, tx_idx integer NOT NULL, raw_log jsonb @@ -709,7 +709,7 @@ ALTER SEQUENCE maker.drip_drip_id_seq OWNED BY maker.drip_drip.id; CREATE TABLE maker.drip_file_ilk ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, vow text, tax numeric, log_idx integer NOT NULL, @@ -931,7 +931,7 @@ ALTER SEQUENCE maker.flop_kick_id_seq OWNED BY maker.flop_kick.id; CREATE TABLE maker.frob ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, dink numeric, dart numeric, @@ -964,6 +964,36 @@ CREATE SEQUENCE maker.frob_id_seq ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id; +-- +-- Name: ilks; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.ilks ( + id integer NOT NULL, + ilk text +); + + +-- +-- Name: ilks_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.ilks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: ilks_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.ilks_id_seq OWNED BY maker.ilks.id; + + -- -- Name: pit_drip; Type: TABLE; Schema: maker; Owner: - -- @@ -1038,7 +1068,7 @@ ALTER SEQUENCE maker.pit_file_debt_ceiling_id_seq OWNED BY maker.pit_file_debt_c CREATE TABLE maker.pit_file_ilk ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, what text, data numeric, log_idx integer NOT NULL, @@ -1075,7 +1105,7 @@ CREATE TABLE maker.pit_ilk_line ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, line numeric NOT NULL ); @@ -1108,7 +1138,7 @@ CREATE TABLE maker.pit_ilk_spot ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, spot numeric NOT NULL ); @@ -1375,7 +1405,7 @@ ALTER SEQUENCE maker.vat_debt_id_seq OWNED BY maker.vat_debt.id; CREATE TABLE maker.vat_flux ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, src text, dst text, rad numeric, @@ -1412,7 +1442,7 @@ ALTER SEQUENCE maker.vat_flux_id_seq OWNED BY maker.vat_flux.id; CREATE TABLE maker.vat_fold ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, rate numeric, log_idx integer NOT NULL, @@ -1449,7 +1479,7 @@ CREATE TABLE maker.vat_gem ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, guy text, gem numeric NOT NULL ); @@ -1482,7 +1512,7 @@ ALTER SEQUENCE maker.vat_gem_id_seq OWNED BY maker.vat_gem.id; CREATE TABLE maker.vat_grab ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, v text, w text, @@ -1558,7 +1588,7 @@ CREATE TABLE maker.vat_ilk_art ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, art numeric NOT NULL ); @@ -1591,7 +1621,7 @@ CREATE TABLE maker.vat_ilk_ink ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, ink numeric NOT NULL ); @@ -1624,7 +1654,7 @@ CREATE TABLE maker.vat_ilk_rate ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, rate numeric NOT NULL ); @@ -1657,7 +1687,7 @@ CREATE TABLE maker.vat_ilk_take ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, take numeric NOT NULL ); @@ -1689,7 +1719,7 @@ ALTER SEQUENCE maker.vat_ilk_take_id_seq OWNED BY maker.vat_ilk_take.id; CREATE TABLE maker.vat_init ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, log_idx integer NOT NULL, tx_idx integer NOT NULL, raw_log jsonb @@ -1792,7 +1822,7 @@ ALTER SEQUENCE maker.vat_sin_id_seq OWNED BY maker.vat_sin.id; CREATE TABLE maker.vat_slip ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, guy text, rad numeric, tx_idx integer NOT NULL, @@ -1828,7 +1858,7 @@ ALTER SEQUENCE maker.vat_slip_id_seq OWNED BY maker.vat_slip.id; CREATE TABLE maker.vat_toll ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, take numeric, tx_idx integer NOT NULL, @@ -1864,7 +1894,7 @@ ALTER SEQUENCE maker.vat_toll_id_seq OWNED BY maker.vat_toll.id; CREATE TABLE maker.vat_tune ( id integer NOT NULL, header_id integer NOT NULL, - ilk text, + ilk integer NOT NULL, urn text, v text, w text, @@ -1904,7 +1934,7 @@ CREATE TABLE maker.vat_urn_art ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, urn text, art text ); @@ -1938,7 +1968,7 @@ CREATE TABLE maker.vat_urn_ink ( id integer NOT NULL, block_number bigint, block_hash text, - ilk text, + ilk integer NOT NULL, urn text, ink numeric NOT NULL ); @@ -3019,6 +3049,13 @@ ALTER TABLE ONLY maker.flop_kick ALTER COLUMN id SET DEFAULT nextval('maker.flop ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_seq'::regclass); +-- +-- Name: ilks id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.ilks ALTER COLUMN id SET DEFAULT nextval('maker.ilks_id_seq'::regclass); + + -- -- Name: pit_drip id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -3710,6 +3747,22 @@ ALTER TABLE ONLY maker.frob ADD CONSTRAINT frob_pkey PRIMARY KEY (id); +-- +-- Name: ilks ilks_ilk_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.ilks + ADD CONSTRAINT ilks_ilk_key UNIQUE (ilk); + + +-- +-- Name: ilks ilks_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.ilks + ADD CONSTRAINT ilks_pkey PRIMARY KEY (id); + + -- -- Name: pit_drip pit_drip_pkey; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -4320,6 +4373,14 @@ ALTER TABLE ONLY maker.bite ADD CONSTRAINT bite_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: bite bite_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.bite + ADD CONSTRAINT bite_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: cat_file_chop_lump cat_file_chop_lump_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4328,6 +4389,14 @@ ALTER TABLE ONLY maker.cat_file_chop_lump ADD CONSTRAINT cat_file_chop_lump_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: cat_file_chop_lump cat_file_chop_lump_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.cat_file_chop_lump + ADD CONSTRAINT cat_file_chop_lump_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: cat_file_flip cat_file_flip_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4368,6 +4437,14 @@ ALTER TABLE ONLY maker.drip_drip ADD CONSTRAINT drip_drip_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: drip_drip drip_drip_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.drip_drip + ADD CONSTRAINT drip_drip_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: drip_file_ilk drip_file_ilk_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4376,6 +4453,14 @@ ALTER TABLE ONLY maker.drip_file_ilk ADD CONSTRAINT drip_file_ilk_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: drip_file_ilk drip_file_ilk_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.drip_file_ilk + ADD CONSTRAINT drip_file_ilk_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: drip_file_repo drip_file_repo_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4424,6 +4509,14 @@ ALTER TABLE ONLY maker.frob ADD CONSTRAINT frob_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: frob frob_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.frob + ADD CONSTRAINT frob_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: pit_file_debt_ceiling pit_file_debt_ceiling_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4440,6 +4533,30 @@ ALTER TABLE ONLY maker.pit_file_ilk ADD CONSTRAINT pit_file_ilk_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: pit_file_ilk pit_file_ilk_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_file_ilk + ADD CONSTRAINT pit_file_ilk_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: pit_ilk_line pit_ilk_line_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_line + ADD CONSTRAINT pit_ilk_line_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: pit_ilk_spot pit_ilk_spot_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pit_ilk_spot + ADD CONSTRAINT pit_ilk_spot_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: price_feeds price_feeds_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4464,6 +4581,14 @@ ALTER TABLE ONLY maker.vat_flux ADD CONSTRAINT vat_flux_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_flux vat_flux_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_flux + ADD CONSTRAINT vat_flux_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_fold vat_fold_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4472,6 +4597,22 @@ ALTER TABLE ONLY maker.vat_fold ADD CONSTRAINT vat_fold_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_fold vat_fold_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_fold + ADD CONSTRAINT vat_fold_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_gem vat_gem_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_gem + ADD CONSTRAINT vat_gem_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_grab vat_grab_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4480,6 +4621,14 @@ ALTER TABLE ONLY maker.vat_grab ADD CONSTRAINT vat_grab_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_grab vat_grab_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_grab + ADD CONSTRAINT vat_grab_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_heal vat_heal_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4488,6 +4637,38 @@ ALTER TABLE ONLY maker.vat_heal ADD CONSTRAINT vat_heal_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_ilk_art vat_ilk_art_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_ilk_art + ADD CONSTRAINT vat_ilk_art_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_ilk_ink vat_ilk_ink_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_ilk_ink + ADD CONSTRAINT vat_ilk_ink_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_ilk_rate vat_ilk_rate_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_ilk_rate + ADD CONSTRAINT vat_ilk_rate_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_ilk_take vat_ilk_take_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_ilk_take + ADD CONSTRAINT vat_ilk_take_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_init vat_init_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4496,6 +4677,14 @@ ALTER TABLE ONLY maker.vat_init ADD CONSTRAINT vat_init_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_init vat_init_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_init + ADD CONSTRAINT vat_init_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_move vat_move_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4512,6 +4701,14 @@ ALTER TABLE ONLY maker.vat_slip ADD CONSTRAINT vat_slip_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_slip vat_slip_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_slip + ADD CONSTRAINT vat_slip_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_toll vat_toll_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4520,6 +4717,14 @@ ALTER TABLE ONLY maker.vat_toll ADD CONSTRAINT vat_toll_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_toll vat_toll_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_toll + ADD CONSTRAINT vat_toll_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vat_tune vat_tune_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- @@ -4528,6 +4733,30 @@ ALTER TABLE ONLY maker.vat_tune ADD CONSTRAINT vat_tune_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_tune vat_tune_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_tune + ADD CONSTRAINT vat_tune_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_urn_art vat_urn_art_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_urn_art + ADD CONSTRAINT vat_urn_art_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + +-- +-- Name: vat_urn_ink vat_urn_ink_ilk_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_urn_ink + ADD CONSTRAINT vat_urn_ink_ilk_fkey FOREIGN KEY (ilk) REFERENCES maker.ilks(id); + + -- -- Name: vow_flog vow_flog_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 118831a4..927dc032 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -50,11 +50,20 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er return fmt.Errorf("model of type %T, not %T", model, BiteModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(biteModel.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, ink = $4, art = $5, iart = $6, tab = $7, nflip = $8, raw_log = $11;`, - headerID, biteModel.Ilk, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw, + headerID, ilkID, biteModel.Urn, biteModel.Ink, biteModel.Art, biteModel.IArt, biteModel.Tab, biteModel.NFlip, biteModel.LogIndex, biteModel.TransactionIndex, biteModel.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/bite/repository_test.go b/pkg/transformers/bite/repository_test.go index 406e9c08..dc0f2b0c 100644 --- a/pkg/transformers/bite/repository_test.go +++ b/pkg/transformers/bite/repository_test.go @@ -19,6 +19,8 @@ package bite_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -67,7 +69,9 @@ var _ = Describe("Bite repository", func() { var dbBite bite.BiteModel err = db.Get(&dbBite, `SELECT ilk, urn, ink, art, tab, nflip, iart, log_idx, tx_idx, raw_log FROM maker.bite WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbBite.Ilk).To(Equal(test_data.BiteModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.BiteModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbBite.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbBite.Urn).To(Equal(test_data.BiteModel.Urn)) Expect(dbBite.Ink).To(Equal(test_data.BiteModel.Ink)) Expect(dbBite.Art).To(Equal(test_data.BiteModel.Art)) diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index e8ebea51..4cc68b2e 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -47,11 +47,20 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte return fmt.Errorf("model of type %T, not %T", model, CatFileChopLumpModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(chopLump.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`, - headerID, chopLump.Ilk, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw, + headerID, ilkID, chopLump.What, chopLump.Data, chopLump.TransactionIndex, chopLump.LogIndex, chopLump.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/cat_file/chop_lump/repository_test.go b/pkg/transformers/cat_file/chop_lump/repository_test.go index eade438e..7747a86b 100644 --- a/pkg/transformers/cat_file/chop_lump/repository_test.go +++ b/pkg/transformers/cat_file/chop_lump/repository_test.go @@ -23,10 +23,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/cat_file/chop_lump" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" ) var _ = Describe("Cat file chop lump repository", func() { @@ -65,7 +67,9 @@ var _ = Describe("Cat file chop lump repository", func() { var dbResult chop_lump.CatFileChopLumpModel err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbResult.Ilk).To(Equal(test_data.CatFileChopModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.CatFileChopModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult.What).To(Equal(test_data.CatFileChopModel.What)) Expect(dbResult.Data).To(Equal(test_data.CatFileChopModel.Data)) Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileChopModel.TransactionIndex)) @@ -83,7 +87,9 @@ var _ = Describe("Cat file chop lump repository", func() { var dbResult chop_lump.CatFileChopLumpModel err = db.Get(&dbResult, `SELECT ilk, what, data, tx_idx, log_idx, raw_log FROM maker.cat_file_chop_lump WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbResult.Ilk).To(Equal(test_data.CatFileLumpModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.CatFileLumpModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult.What).To(Equal(test_data.CatFileLumpModel.What)) Expect(dbResult.Data).To(Equal(test_data.CatFileLumpModel.Data)) Expect(dbResult.TransactionIndex).To(Equal(test_data.CatFileLumpModel.TransactionIndex)) diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index 0c21b060..accf41f6 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -46,11 +46,20 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} return fmt.Errorf("model of type %T, not %T", model, DripDripModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(dripDrip.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk= $2, raw_log = $5;`, - headerID, dripDrip.Ilk, dripDrip.LogIndex, dripDrip.TransactionIndex, dripDrip.Raw, + headerID, ilkID, dripDrip.LogIndex, dripDrip.TransactionIndex, dripDrip.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/drip_drip/repository_test.go b/pkg/transformers/drip_drip/repository_test.go index 6dbed2cc..c97e477c 100644 --- a/pkg/transformers/drip_drip/repository_test.go +++ b/pkg/transformers/drip_drip/repository_test.go @@ -24,10 +24,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" ) var _ = Describe("Drip drip repository", func() { @@ -67,7 +69,9 @@ var _ = Describe("Drip drip repository", func() { var dbDripDrip drip_drip.DripDripModel err = db.Get(&dbDripDrip, `SELECT ilk, log_idx, tx_idx, raw_log FROM maker.drip_drip WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbDripDrip.Ilk).To(Equal(test_data.DripDripModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.DripDripModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbDripDrip.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbDripDrip.LogIndex).To(Equal(test_data.DripDripModel.LogIndex)) Expect(dbDripDrip.TransactionIndex).To(Equal(test_data.DripDripModel.TransactionIndex)) Expect(dbDripDrip.Raw).To(MatchJSON(test_data.DripDripModel.Raw)) diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index dc7eeacd..b0122520 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -47,10 +47,19 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(ilk.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, - headerID, ilk.Ilk, ilk.Vow, ilk.Tax, ilk.LogIndex, ilk.TransactionIndex, ilk.Raw, + headerID, ilkID, ilk.Vow, ilk.Tax, ilk.LogIndex, ilk.TransactionIndex, ilk.Raw, ) if execErr != nil { diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 1695f17c..f632e9f8 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -24,10 +24,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" ) var _ = Describe("Drip file ilk repository", func() { @@ -67,7 +69,9 @@ var _ = Describe("Drip file ilk repository", func() { var dbDripFileIlk ilk.DripFileIlkModel err = db.Get(&dbDripFileIlk, `SELECT ilk, vow, tax, log_idx, tx_idx, raw_log FROM maker.drip_file_ilk WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbDripFileIlk.Ilk).To(Equal(test_data.DripFileIlkModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.DripFileIlkModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbDripFileIlk.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbDripFileIlk.Vow).To(Equal(test_data.DripFileIlkModel.Vow)) Expect(dbDripFileIlk.Tax).To(Equal(test_data.DripFileIlkModel.Tax)) Expect(dbDripFileIlk.LogIndex).To(Equal(test_data.DripFileIlkModel.LogIndex)) diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index a8d633c2..14cb20ae 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -45,10 +45,19 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er return fmt.Errorf("model of type %T, not %T", model, FrobModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(frobModel.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx) VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET art = $2, dart = $3, dink = $4, iart = $5, ilk = $6, ink = $7, urn = $8, raw_log = $9;`, - headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) + headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, ilkID, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) if execErr != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { diff --git a/pkg/transformers/frob/repository_test.go b/pkg/transformers/frob/repository_test.go index ae2b0b54..bcb095a4 100644 --- a/pkg/transformers/frob/repository_test.go +++ b/pkg/transformers/frob/repository_test.go @@ -19,6 +19,8 @@ package frob_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -67,7 +69,9 @@ var _ = Describe("Frob repository", func() { err = db.Get(&dbFrob, `SELECT art, dart, dink, iart, ilk, ink, urn, log_idx, tx_idx, raw_log FROM maker.frob WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbFrob.Ilk).To(Equal(test_data.FrobModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.FrobModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbFrob.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbFrob.Urn).To(Equal(test_data.FrobModel.Urn)) Expect(dbFrob.Ink).To(Equal(test_data.FrobModel.Ink)) Expect(dbFrob.Art).To(Equal(test_data.FrobModel.Art)) diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index fdb09140..28c1cb8c 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -19,6 +19,7 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "strconv" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -83,7 +84,9 @@ var _ = Describe("Bite Transformer", func() { Expect(len(dbResult)).To(Equal(1)) Expect(dbResult[0].Art).To(Equal("149846666666666655744")) Expect(dbResult[0].IArt).To(Equal("1645356666666666655736")) - Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Ink).To(Equal("1000000000000000000")) Expect(dbResult[0].NFlip).To(Equal("2")) Expect(dbResult[0].Tab).To(Equal("149846666666666655744")) diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 114dfeaa..48ce0c37 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "sort" + "strconv" "github.com/ethereum/go-ethereum/ethclient" . "github.com/onsi/ginkgo" @@ -97,12 +98,14 @@ var _ = Describe("Cat File transformer", func() { Expect(len(dbResult)).To(Equal(2)) sort.Sort(byLogIndexChopLump(dbResult)) - Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].What).To(Equal("lump")) Expect(dbResult[0].Data).To(Equal("10000.000000000000000000")) Expect(dbResult[0].LogIndex).To(Equal(uint(3))) - Expect(dbResult[1].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + Expect(dbResult[1].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[1].What).To(Equal("chop")) Expect(dbResult[1].Data).To(Equal("1.000000000000000000000000000")) Expect(dbResult[1].LogIndex).To(Equal(uint(4))) diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 927ee6eb..f369a529 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -25,6 +25,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -86,7 +87,9 @@ var _ = Describe("DripDrip Transformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) }) It("rechecks drip drip event", func() { diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index 2a58d878..43ef42d9 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -94,7 +95,9 @@ var _ = Describe("Frob Transformer", func() { Expect(dbResult[0].Dart).To(Equal("0")) Expect(dbResult[0].Dink).To(Equal("10000000000000")) Expect(dbResult[0].IArt).To(Equal("1495509999999999999992")) - Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Ink).To(Equal("10050100000000000")) Expect(dbResult[0].Urn).To(Equal("000000000000000000000000c8e093e5f3f9b5aa6a6b33ea45960b93c161430c")) }) @@ -139,7 +142,9 @@ var _ = Describe("Frob Transformer", func() { Expect(dbResult[0].Dart).To(Equal("0")) Expect(dbResult[0].Dink).To(Equal("10000000000000")) Expect(dbResult[0].IArt).To(Equal("1495509999999999999992")) - Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Ink).To(Equal("10050100000000000")) Expect(dbResult[0].Urn).To(Equal("000000000000000000000000c8e093e5f3f9b5aa6a6b33ea45960b93c161430c")) }) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 5c8f2914..9c8c221c 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -24,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" @@ -87,7 +88,9 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].What).To(Equal("spot")) Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) }) @@ -126,7 +129,9 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("0x4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].What).To(Equal("spot")) Expect(dbResult[0].Data).To(Equal("139.840000000000003410605131648")) }) @@ -158,7 +163,9 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { pitFileIlkLineModel = result } } - Expect(pitFileIlkLineModel.Ilk).To(Equal("0x5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(pitFileIlkLineModel.Data).To(Equal("2000000.000000000000000000")) }) }) diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 8291f763..f2bb9433 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -74,7 +75,9 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Src).To(Equal("000000000000000000000000c0851f73cc8dd5c0765e71980ec7e7fd1ef74434")) Expect(dbResult[0].Dst).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb")) Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) @@ -138,7 +141,9 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Src).To(Equal("000000000000000000000000c0851f73cc8dd5c0765e71980ec7e7fd1ef74434")) Expect(dbResult[0].Dst).To(Equal("0000000000000000000000000000d8b4147eda80fec7122ae16da2479cbd7ffb")) Expect(dbResult[0].Rad).To(Equal("1800000000000000000000000000000000000000000000")) diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index f8dd291e..39bb93f4 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -25,6 +25,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -82,7 +83,9 @@ var _ = Describe("VatFold Transformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult.Urn).To(Equal("0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1")) Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) }) @@ -136,7 +139,9 @@ var _ = Describe("VatFold Transformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult.Urn).To(Equal("0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1")) Expect(dbResult.Rate).To(Equal("0.000000000000000000000000000")) }) diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index fea5af2b..19360710 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -21,6 +21,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "math/big" + "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -75,7 +76,9 @@ var _ = Describe("Vat Grab Transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Urn).To(Equal("0000000000000000000000006a3ae20c315e845b2e398e68effe39139ec6060c")) Expect(dbResult[0].V).To(Equal("0000000000000000000000002f34f22a00ee4b7a8f8bbc4eaee1658774c624e0")) //cat contract address as bytes32 Expect(dbResult[0].W).To(Equal("0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1")) @@ -143,7 +146,9 @@ var _ = Describe("Vat Grab Transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("5245500000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("5245500000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Urn).To(Equal("0000000000000000000000006a3ae20c315e845b2e398e68effe39139ec6060c")) Expect(dbResult[0].V).To(Equal("0000000000000000000000002f34f22a00ee4b7a8f8bbc4eaee1658774c624e0")) //cat contract address Expect(dbResult[0].W).To(Equal("0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1")) diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index 1a9d6db2..0835e396 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -24,6 +24,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" "github.com/vulcanize/vulcanizedb/test_config" @@ -74,7 +75,9 @@ var _ = Describe("VatInit LogNoteTransformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) }) It("rechecks vat init event", func() { @@ -134,6 +137,8 @@ var _ = Describe("VatInit LogNoteTransformer", func() { Expect(len(dbResults)).To(Equal(1)) dbResult := dbResults[0] - Expect(dbResult.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult.Ilk).To(Equal(strconv.Itoa(ilkID))) }) }) diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index fec404b6..f5ba2560 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -66,7 +67,9 @@ var _ = Describe("Vat slip transformer", func() { var model vat_slip.VatSlipModel err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(model.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(model.Guy).To(Equal("000000000000000000000000da15dce70ab462e66779f23ee14f21d993789ee3")) Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000")) Expect(model.TransactionIndex).To(Equal(uint(0))) @@ -124,7 +127,9 @@ var _ = Describe("Vat slip transformer", func() { var model vat_slip.VatSlipModel err = db.Get(&model, `SELECT ilk, guy, rad, tx_idx FROM maker.vat_slip WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(model.Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(model.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(model.Guy).To(Equal("000000000000000000000000da15dce70ab462e66779f23ee14f21d993789ee3")) Expect(model.Rad).To(Equal("100000000000000000000000000000000000000000000000")) Expect(model.TransactionIndex).To(Equal(uint(0))) diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index 3aaad692..c962f278 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -21,6 +21,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "math/big" + "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -75,7 +76,9 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Urn).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) Expect(dbResult[0].V).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) Expect(dbResult[0].W).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) @@ -142,7 +145,9 @@ var _ = Describe("VatTune LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(1)) - Expect(dbResult[0].Ilk).To(Equal("4554480000000000000000000000000000000000000000000000000000000000")) + ilkID, err := shared.GetOrCreateIlk("4554480000000000000000000000000000000000000000000000000000000000", db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Urn).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) Expect(dbResult[0].V).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) Expect(dbResult[0].W).To(Equal("0000000000000000000000004f26ffbe5f04ed43630fdc30a87638d53d0b0876")) diff --git a/pkg/transformers/pit_file/ilk/converter.go b/pkg/transformers/pit_file/ilk/converter.go index 3945f22b..3013b032 100644 --- a/pkg/transformers/pit_file/ilk/converter.go +++ b/pkg/transformers/pit_file/ilk/converter.go @@ -37,7 +37,7 @@ func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) if err != nil { return nil, err } - ilk := ethLog.Topics[2].Hex() + ilk := shared.GetHexWithoutPrefix(ethLog.Topics[2].Bytes()) what := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) dataBytes := ethLog.Data[len(ethLog.Data)-constants.DataItemLength:] data, err := getData(dataBytes, what) diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 90bd0cfc..25b5af34 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -47,11 +47,20 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(pitFileIlk.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, what = $3, data = $4, raw_log = $7;`, - headerID, pitFileIlk.Ilk, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.LogIndex, pitFileIlk.TransactionIndex, pitFileIlk.Raw, + headerID, ilkID, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.LogIndex, pitFileIlk.TransactionIndex, pitFileIlk.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index 99ea098e..f4b79681 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -19,6 +19,8 @@ package ilk_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -67,7 +69,9 @@ var _ = Describe("Pit file ilk repository", func() { var dbPitFile ilk.PitFileIlkModel err = db.Get(&dbPitFile, `SELECT ilk, what, data, log_idx, tx_idx, raw_log FROM maker.pit_file_ilk WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbPitFile.Ilk).To(Equal(test_data.PitFileIlkSpotModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.PitFileIlkSpotModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbPitFile.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbPitFile.What).To(Equal(test_data.PitFileIlkSpotModel.What)) Expect(dbPitFile.Data).To(Equal(test_data.PitFileIlkSpotModel.Data)) Expect(dbPitFile.LogIndex).To(Equal(test_data.PitFileIlkSpotModel.LogIndex)) diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 834e78ac..bf156067 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -10,6 +10,30 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" ) +func GetOrCreateIlk(ilk string, db *postgres.DB) (int, error) { + var ilkID int + err := db.Get(&ilkID, `SELECT id FROM maker.ilks WHERE ilk = $1`, ilk) + if err != nil { + if err == sql.ErrNoRows { + insertErr := db.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID) + return ilkID, insertErr + } + } + return ilkID, err +} + +func GetOrCreateIlkInTransaction(ilk string, tx *sql.Tx) (int, error) { + var ilkID int + err := tx.QueryRow(`SELECT id FROM maker.ilks WHERE ilk = $1`, ilk).Scan(&ilkID) + if err != nil { + if err == sql.ErrNoRows { + insertErr := tx.QueryRow(`INSERT INTO maker.ilks (ilk) VALUES ($1) RETURNING id`, ilk).Scan(&ilkID) + return ilkID, insertErr + } + } + return ilkID, err +} + func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) diff --git a/pkg/transformers/storage_diffs/maker/maker_storage_repository.go b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go index 2615d3f7..7916b059 100644 --- a/pkg/transformers/storage_diffs/maker/maker_storage_repository.go +++ b/pkg/transformers/storage_diffs/maker/maker_storage_repository.go @@ -66,19 +66,36 @@ func (repository *MakerStorageRepository) GetMaxFlip() (*big.Int, error) { func (repository *MakerStorageRepository) GetGemKeys() ([]Urn, error) { var gems []Urn err := repository.db.Select(&gems, ` - SELECT DISTINCT ilk, guy FROM maker.vat_slip UNION - SELECT DISTINCT ilk, src AS guy FROM maker.vat_flux UNION - SELECT DISTINCT ilk, dst AS guy FROM maker.vat_flux UNION - SELECT DISTINCT ilk, v AS guy FROM maker.vat_tune UNION - SELECT DISTINCT ilk, v AS guy FROM maker.vat_grab UNION - SELECT DISTINCT ilk, urn AS guy FROM maker.vat_toll + SELECT DISTINCT ilks.ilk, slip.guy + FROM maker.vat_slip slip + INNER JOIN maker.ilks ilks ON ilks.id = slip.ilk + UNION + SELECT DISTINCT ilks.ilk, flux.src AS guy + FROM maker.vat_flux flux + INNER JOIN maker.ilks ilks ON ilks.id = flux.ilk + UNION + SELECT DISTINCT ilks.ilk, flux.dst AS guy + FROM maker.vat_flux flux + INNER JOIN maker.ilks ilks ON ilks.id = flux.ilk + UNION + SELECT DISTINCT ilks.ilk, tune.v AS guy + FROM maker.vat_tune tune + INNER JOIN maker.ilks ilks ON ilks.id = tune.ilk + UNION + SELECT DISTINCT ilks.ilk, grab.v AS guy + FROM maker.vat_grab grab + INNER JOIN maker.ilks ilks ON ilks.id = grab.ilk + UNION + SELECT DISTINCT ilks.ilk, toll.urn AS guy + FROM maker.vat_toll toll + INNER JOIN maker.ilks ilks ON ilks.id = toll.ilk `) return gems, err } func (repository MakerStorageRepository) GetIlks() ([]string, error) { var ilks []string - err := repository.db.Select(&ilks, `SELECT DISTINCT ilk FROM maker.vat_init`) + err := repository.db.Select(&ilks, `SELECT DISTINCT ilk FROM maker.ilks`) return ilks, err } @@ -91,8 +108,15 @@ func (repository *MakerStorageRepository) GetSinKeys() ([]string, error) { func (repository *MakerStorageRepository) GetUrns() ([]Urn, error) { var urns []Urn - err := repository.db.Select(&urns, `SELECT DISTINCT ilk, urn AS guy FROM maker.vat_tune UNION - SELECT DISTINCT ilk, urn AS guy FROM maker.vat_grab`) + err := repository.db.Select(&urns, ` + SELECT DISTINCT ilks.ilk, tune.urn AS guy + FROM maker.vat_tune tune + INNER JOIN maker.ilks ilks ON ilks.id = tune.ilk + UNION + SELECT DISTINCT ilks.ilk, grab.urn AS guy + FROM maker.vat_grab grab + INNER JOIN maker.ilks ilks ON ilks.id = grab.ilk +`) return urns, err } diff --git a/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go b/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go index 013007dc..ac8a5e68 100644 --- a/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go +++ b/pkg/transformers/storage_diffs/maker/maker_storage_repository_test.go @@ -22,6 +22,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -289,10 +290,12 @@ func insertVatFold(urn string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk("fake_ilk", db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( - `INSERT INTO maker.vat_fold (header_id, urn, log_idx, tx_idx) - VALUES($1, $2, $3, $4)`, - headerID, urn, 0, 0, + `INSERT INTO maker.vat_fold (header_id, urn, ilk, log_idx, tx_idx) + VALUES($1, $2, $3, $4, $5)`, + headerID, urn, ilkID, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -301,10 +304,12 @@ func insertVatFlux(ilk, src, dst string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_flux (header_id, ilk, src, dst, log_idx, tx_idx) VALUES($1, $2, $3, $4, $5, $6)`, - headerID, ilk, src, dst, 0, 0, + headerID, ilkID, src, dst, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -313,10 +318,12 @@ func insertVatGrab(ilk, urn, v, w string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_grab (header_id, ilk, urn, v, w, log_idx, tx_idx) VALUES($1, $2, $3, $4, $5, $6, $7)`, - headerID, ilk, urn, v, w, 0, 0, + headerID, ilkID, urn, v, w, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -337,10 +344,12 @@ func insertVatInit(ilk string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx) VALUES($1, $2, $3, $4)`, - headerID, ilk, 0, 0, + headerID, ilkID, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -361,10 +370,12 @@ func insertVatSlip(ilk, guy string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_slip (header_id, ilk, guy, log_idx, tx_idx) VALUES($1, $2, $3, $4, $5)`, - headerID, ilk, guy, 0, 0, + headerID, ilkID, guy, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -373,10 +384,12 @@ func insertVatToll(ilk, urn string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_toll (header_id, ilk, urn, log_idx, tx_idx) VALUES($1, $2, $3, $4, $5)`, - headerID, ilk, urn, 0, 0, + headerID, ilkID, urn, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } @@ -385,10 +398,12 @@ func insertVatTune(ilk, urn, v, w string, blockNumber int64, db *postgres.DB) { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber)) Expect(err).NotTo(HaveOccurred()) + ilkID, err := shared.GetOrCreateIlk(ilk, db) + Expect(err).NotTo(HaveOccurred()) _, execErr := db.Exec( `INSERT INTO maker.vat_tune (header_id, ilk, urn, v, w, log_idx, tx_idx) VALUES($1, $2, $3, $4, $5, $6, $7)`, - headerID, ilk, urn, v, w, 0, 0, + headerID, ilkID, urn, v, w, 0, 0, ) Expect(execErr).NotTo(HaveOccurred()) } diff --git a/pkg/transformers/storage_diffs/maker/pit/repository.go b/pkg/transformers/storage_diffs/maker/pit/repository.go index c11212cb..1c748b6d 100644 --- a/pkg/transformers/storage_diffs/maker/pit/repository.go +++ b/pkg/transformers/storage_diffs/maker/pit/repository.go @@ -19,6 +19,7 @@ package pit import ( "fmt" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" ) @@ -50,13 +51,51 @@ func (repository PitStorageRepository) Create(blockNumber int, blockHash string, } func (repository PitStorageRepository) insertIlkLine(blockNumber int, blockHash string, ilk string, line string) error { - _, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_line (block_number, block_hash, ilk, line) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, line) - return err + tx, err := repository.db.Begin() + if err != nil { + return err + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.pit_ilk_line (block_number, block_hash, ilk, line) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, line) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert pit ilk line: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository PitStorageRepository) insertIlkSpot(blockNumber int, blockHash string, ilk string, spot string) error { - _, err := repository.db.Exec(`INSERT INTO maker.pit_ilk_spot (block_number, block_hash, ilk, spot) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, spot) - return err + tx, err := repository.db.Begin() + if err != nil { + return err + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.pit_ilk_spot (block_number, block_hash, ilk, spot) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, spot) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert pit ilk spot: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository PitStorageRepository) insertPitDrip(blockNumber int, blockHash string, drip string) error { diff --git a/pkg/transformers/storage_diffs/maker/pit/repository_test.go b/pkg/transformers/storage_diffs/maker/pit/repository_test.go index 3b9e6ddf..3e07c526 100644 --- a/pkg/transformers/storage_diffs/maker/pit/repository_test.go +++ b/pkg/transformers/storage_diffs/maker/pit/repository_test.go @@ -20,10 +20,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/pit" . "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" ) var _ = Describe("Pit storage repository", func() { @@ -59,7 +61,9 @@ var _ = Describe("Pit storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, line AS value FROM maker.pit_ilk_line`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, blockNumber, blockHash, expectedIlk, expectedLine) + ilkID, err := shared2.GetOrCreateIlk(expectedIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, blockNumber, blockHash, strconv.Itoa(ilkID), expectedLine) }) It("persists an ilk spot", func() { @@ -77,7 +81,9 @@ var _ = Describe("Pit storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, spot AS value FROM maker.pit_ilk_spot`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, blockNumber, blockHash, expectedIlk, expectedSpot) + ilkID, err := shared2.GetOrCreateIlk(expectedIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, blockNumber, blockHash, strconv.Itoa(ilkID), expectedSpot) }) It("persists a pit drip", func() { diff --git a/pkg/transformers/storage_diffs/maker/vat/repository.go b/pkg/transformers/storage_diffs/maker/vat/repository.go index d237b352..65dcf5b9 100644 --- a/pkg/transformers/storage_diffs/maker/vat/repository.go +++ b/pkg/transformers/storage_diffs/maker/vat/repository.go @@ -3,6 +3,7 @@ package vat import ( "fmt" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" ) @@ -53,16 +54,35 @@ func (repository *VatStorageRepository) insertDai(blockNumber int, blockHash str } func (repository *VatStorageRepository) insertGem(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, gem string) error { - ilk, ilkErr := getIlk(metadata.Keys) - if ilkErr != nil { - return ilkErr + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err } guy, guyErr := getGuy(metadata.Keys) if guyErr != nil { return guyErr } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_gem (block_number, block_hash, ilk, guy, gem) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilk, guy, gem) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_gem (block_number, block_hash, ilk, guy, gem) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilkID, guy, gem) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert gem: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertIlkArt(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, art string) error { @@ -70,8 +90,27 @@ func (repository *VatStorageRepository) insertIlkArt(blockNumber int, blockHash if err != nil { return err } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_ilk_art (block_number, block_hash, ilk, art) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, art) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_ilk_art (block_number, block_hash, ilk, art) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, art) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk art: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertIlkInk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ink string) error { @@ -79,8 +118,27 @@ func (repository *VatStorageRepository) insertIlkInk(blockNumber int, blockHash if err != nil { return err } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_ilk_ink (block_number, block_hash, ilk, ink) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, ink) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_ilk_ink (block_number, block_hash, ilk, ink) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, ink) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk ink: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertIlkRate(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, rate string) error { @@ -88,8 +146,27 @@ func (repository *VatStorageRepository) insertIlkRate(blockNumber int, blockHash if err != nil { return err } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_ilk_rate (block_number, block_hash, ilk, rate) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, rate) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_ilk_rate (block_number, block_hash, ilk, rate) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, rate) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk rate: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertIlkTake(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, take string) error { @@ -97,8 +174,27 @@ func (repository *VatStorageRepository) insertIlkTake(blockNumber int, blockHash if err != nil { return err } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_ilk_take (block_number, block_hash, ilk, take) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilk, take) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_ilk_take (block_number, block_hash, ilk, take) VALUES ($1, $2, $3, $4)`, blockNumber, blockHash, ilkID, take) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk take: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertSin(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, sin string) error { @@ -111,29 +207,67 @@ func (repository *VatStorageRepository) insertSin(blockNumber int, blockHash str } func (repository *VatStorageRepository) insertUrnArt(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, art string) error { - ilk, ilkErr := getIlk(metadata.Keys) - if ilkErr != nil { - return ilkErr + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err } guy, guyErr := getGuy(metadata.Keys) if guyErr != nil { return guyErr } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_urn_art (block_number, block_hash, ilk, urn, art) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilk, guy, art) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_urn_art (block_number, block_hash, ilk, urn, art) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilkID, guy, art) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert urn art: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertUrnInk(blockNumber int, blockHash string, metadata shared.StorageValueMetadata, ink string) error { - ilk, ilkErr := getIlk(metadata.Keys) - if ilkErr != nil { - return ilkErr + ilk, err := getIlk(metadata.Keys) + if err != nil { + return err } guy, guyErr := getGuy(metadata.Keys) if guyErr != nil { return guyErr } - _, writeErr := repository.db.Exec(`INSERT INTO maker.vat_urn_ink (block_number, block_hash, ilk, urn, ink) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilk, guy, ink) - return writeErr + tx, txErr := repository.db.Begin() + if txErr != nil { + return txErr + } + ilkID, ilkErr := shared2.GetOrCreateIlkInTransaction(ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert ilk: %s", ilkErr.Error()) + } + return ilkErr + } + _, writeErr := tx.Exec(`INSERT INTO maker.vat_urn_ink (block_number, block_hash, ilk, urn, ink) VALUES ($1, $2, $3, $4, $5)`, blockNumber, blockHash, ilkID, guy, ink) + if writeErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return fmt.Errorf("failed to rollback transaction after failing to insert urn ink: %s", writeErr.Error()) + } + return writeErr + } + return tx.Commit() } func (repository *VatStorageRepository) insertVatDebt(blockNumber int, blockHash, debt string) error { diff --git a/pkg/transformers/storage_diffs/maker/vat/repository_test.go b/pkg/transformers/storage_diffs/maker/vat/repository_test.go index a139c132..1c9788d9 100644 --- a/pkg/transformers/storage_diffs/maker/vat/repository_test.go +++ b/pkg/transformers/storage_diffs/maker/vat/repository_test.go @@ -4,10 +4,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" . "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/maker/vat" "github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared" "github.com/vulcanize/vulcanizedb/test_config" + "strconv" ) var _ = Describe("Vat storage repository", func() { @@ -75,7 +77,9 @@ var _ = Describe("Vat storage repository", func() { var result DoubleMappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key_one, guy AS key_two, gem AS value FROM maker.vat_gem`) Expect(err).NotTo(HaveOccurred()) - AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeGuy, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeGuy, fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -120,7 +124,9 @@ var _ = Describe("Vat storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, art AS value FROM maker.vat_ilk_art`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -152,7 +158,9 @@ var _ = Describe("Vat storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, ink AS value FROM maker.vat_ilk_ink`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -184,7 +192,9 @@ var _ = Describe("Vat storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, rate AS value FROM maker.vat_ilk_rate`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -216,7 +226,9 @@ var _ = Describe("Vat storage repository", func() { var result MappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key, take AS value FROM maker.vat_ilk_take`) Expect(err).NotTo(HaveOccurred()) - AssertMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -280,7 +292,9 @@ var _ = Describe("Vat storage repository", func() { var result DoubleMappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key_one, urn AS key_two, art AS value FROM maker.vat_urn_art`) Expect(err).NotTo(HaveOccurred()) - AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeGuy, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeGuy, fakeUint256) }) It("returns error if metadata missing ilk", func() { @@ -325,7 +339,9 @@ var _ = Describe("Vat storage repository", func() { var result DoubleMappingRes err = db.Get(&result, `SELECT block_number, block_hash, ilk AS key_one, urn AS key_two, ink AS value FROM maker.vat_urn_ink`) Expect(err).NotTo(HaveOccurred()) - AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, fakeIlk, fakeGuy, fakeUint256) + ilkID, err := shared2.GetOrCreateIlk(fakeIlk, db) + Expect(err).NotTo(HaveOccurred()) + AssertDoubleMapping(result, fakeBlockNumber, fakeBlockHash, strconv.Itoa(ilkID), fakeGuy, fakeUint256) }) It("returns error if metadata missing ilk", func() { diff --git a/pkg/transformers/test_data/pit_file.go b/pkg/transformers/test_data/pit_file.go index 38895da9..155578b5 100644 --- a/pkg/transformers/test_data/pit_file.go +++ b/pkg/transformers/test_data/pit_file.go @@ -75,7 +75,7 @@ var EthPitFileIlkLineLog = types.Log{ var rawPitFileIlkLineLog, _ = json.Marshal(EthPitFileIlkLineLog) var PitFileIlkLineModel = ilk2.PitFileIlkModel{ - Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", What: "line", Data: "0.000001000000000000", LogIndex: EthPitFileIlkLineLog.Index, @@ -102,7 +102,7 @@ var EthPitFileIlkSpotLog = types.Log{ var rawPitFileIlkSpotLog, _ = json.Marshal(EthPitFileIlkSpotLog) var PitFileIlkSpotModel = ilk2.PitFileIlkModel{ - Ilk: "0x66616b6520696c6b000000000000000000000000000000000000000000000000", + Ilk: "66616b6520696c6b000000000000000000000000000000000000000000000000", What: "spot", Data: "0.000000000000001000000000000", LogIndex: EthPitFileIlkSpotLog.Index, diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index 5e299ed1..516405e2 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -47,10 +47,19 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatFluxModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatFlux.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, dst = $3, src = $4, rad = $5, raw_log = $8;`, - headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) + headerID, ilkID, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) if execErr != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go index fb6cfd41..8bbde4ba 100644 --- a/pkg/transformers/vat_flux/repository_test.go +++ b/pkg/transformers/vat_flux/repository_test.go @@ -19,6 +19,8 @@ package vat_flux_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -76,7 +78,9 @@ var _ = Describe("VatFlux Repository", func() { err = db.Select(&dbResult, `SELECT * from maker.vat_flux where header_id = $1`, headerId) Expect(err).NotTo(HaveOccurred()) Expect(len(dbResult)).To(Equal(2)) - Expect(dbResult[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatFluxModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbResult[0].Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbResult[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) Expect(dbResult[0].Src).To(Equal(test_data.VatFluxModel.Src)) Expect(dbResult[0].Rad).To(Equal(test_data.VatFluxModel.Rad)) diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 2a199172..96fa1c44 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -45,11 +45,20 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatFoldModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatFold.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, rate = $4, raw_log = $7;`, - headerID, vatFold.Ilk, vatFold.Urn, vatFold.Rate, vatFold.LogIndex, vatFold.TransactionIndex, vatFold.Raw, + headerID, ilkID, vatFold.Urn, vatFold.Rate, vatFold.LogIndex, vatFold.TransactionIndex, vatFold.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index dbd15a4d..8ac0f679 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -19,6 +19,8 @@ package vat_fold_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -69,7 +71,9 @@ var _ = Describe("Vat.fold repository", func() { err = db.Get(&dbVatFold, `SELECT ilk, urn, rate, log_idx, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatFold.Ilk).To(Equal(test_data.VatFoldModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatFoldModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatFold.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatFold.Urn).To(Equal(test_data.VatFoldModel.Urn)) Expect(dbVatFold.Rate).To(Equal(test_data.VatFoldModel.Rate)) Expect(dbVatFold.LogIndex).To(Equal(test_data.VatFoldModel.LogIndex)) diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index 24d1fabb..80dbcfdd 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -29,11 +29,20 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatGrab.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`, - headerID, vatGrab.Ilk, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw, + headerID, ilkID, vatGrab.Urn, vatGrab.V, vatGrab.W, vatGrab.Dink, vatGrab.Dart, vatGrab.LogIndex, vatGrab.TransactionIndex, vatGrab.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_grab/repository_test.go b/pkg/transformers/vat_grab/repository_test.go index e5dded85..ac77c075 100644 --- a/pkg/transformers/vat_grab/repository_test.go +++ b/pkg/transformers/vat_grab/repository_test.go @@ -3,6 +3,8 @@ package vat_grab_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -53,7 +55,9 @@ var _ = Describe("Vat grab repository", func() { var dbVatGrab vat_grab.VatGrabModel err = db.Get(&dbVatGrab, `SELECT ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log FROM maker.vat_grab WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatGrab.Ilk).To(Equal(test_data.VatGrabModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatGrabModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatGrab.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatGrab.Urn).To(Equal(test_data.VatGrabModel.Urn)) Expect(dbVatGrab.V).To(Equal(test_data.VatGrabModel.V)) Expect(dbVatGrab.W).To(Equal(test_data.VatGrabModel.W)) diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index d56c3bcb..0c8bcb8a 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -47,11 +47,20 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatInitModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatInit.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, raw_log = $5;`, - headerID, vatInit.Ilk, vatInit.LogIndex, vatInit.TransactionIndex, vatInit.Raw, + headerID, ilkID, vatInit.LogIndex, vatInit.TransactionIndex, vatInit.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 6b9ff3b6..36142337 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -19,8 +19,10 @@ package vat_init_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -67,7 +69,9 @@ var _ = Describe("Vat init repository", func() { var dbVatInit vat_init.VatInitModel err = db.Get(&dbVatInit, `SELECT ilk, log_idx, tx_idx, raw_log FROM maker.vat_init WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatInit.Ilk).To(Equal(test_data.VatInitModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatInitModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatInit.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatInit.LogIndex).To(Equal(test_data.VatInitModel.LogIndex)) Expect(dbVatInit.TransactionIndex).To(Equal(test_data.VatInitModel.TransactionIndex)) Expect(dbVatInit.Raw).To(MatchJSON(test_data.VatInitModel.Raw)) diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index 40f1fb57..5ca09c93 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -30,11 +30,20 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatSlipModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatSlip.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, guy = $3, rad = $4, raw_log = $7;`, - headerID, vatSlip.Ilk, vatSlip.Guy, vatSlip.Rad, vatSlip.TransactionIndex, vatSlip.LogIndex, vatSlip.Raw, + headerID, ilkID, vatSlip.Guy, vatSlip.Rad, vatSlip.TransactionIndex, vatSlip.LogIndex, vatSlip.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_slip/repository_test.go b/pkg/transformers/vat_slip/repository_test.go index 23a83150..1c6d6371 100644 --- a/pkg/transformers/vat_slip/repository_test.go +++ b/pkg/transformers/vat_slip/repository_test.go @@ -3,8 +3,10 @@ package vat_slip_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" + "strconv" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -51,7 +53,9 @@ var _ = Describe("Vat slip repository", func() { var dbVatSlip vat_slip.VatSlipModel err = db.Get(&dbVatSlip, `SELECT ilk, guy, rad, tx_idx, log_idx, raw_log FROM maker.vat_slip WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatSlip.Ilk).To(Equal(test_data.VatSlipModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatSlipModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatSlip.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatSlip.Guy).To(Equal(test_data.VatSlipModel.Guy)) Expect(dbVatSlip.Rad).To(Equal(test_data.VatSlipModel.Rad)) Expect(dbVatSlip.TransactionIndex).To(Equal(test_data.VatSlipModel.TransactionIndex)) diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 1af342b4..425e3301 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -30,10 +30,19 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatTollModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatToll.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, - headerID, vatToll.Ilk, vatToll.Urn, vatToll.Take, vatToll.TransactionIndex, vatToll.LogIndex, vatToll.Raw, + headerID, ilkID, vatToll.Urn, vatToll.Take, vatToll.TransactionIndex, vatToll.LogIndex, vatToll.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_toll/repository_test.go b/pkg/transformers/vat_toll/repository_test.go index 3c2f9f0d..fd8b228d 100644 --- a/pkg/transformers/vat_toll/repository_test.go +++ b/pkg/transformers/vat_toll/repository_test.go @@ -1,8 +1,10 @@ package vat_toll_test import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" + "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -51,7 +53,9 @@ var _ = Describe("Vat toll repository", func() { var dbVatToll vat_toll.VatTollModel err = db.Get(&dbVatToll, `SELECT ilk, urn, take, tx_idx, log_idx, raw_log FROM maker.vat_toll WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatToll.Ilk).To(Equal(test_data.VatTollModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatTollModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatToll.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatToll.Urn).To(Equal(test_data.VatTollModel.Urn)) Expect(dbVatToll.Take).To(Equal(test_data.VatTollModel.Take)) Expect(dbVatToll.TransactionIndex).To(Equal(test_data.VatTollModel.TransactionIndex)) diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index 99855bb0..e7ea9228 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -30,11 +30,20 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) return fmt.Errorf("model of type %T, not %T", model, VatTuneModel{}) } + ilkID, ilkErr := shared.GetOrCreateIlkInTransaction(vatTune.Ilk, tx) + if ilkErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback ", rollbackErr) + } + return ilkErr + } + _, execErr := tx.Exec( `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10) ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET ilk = $2, urn = $3, v = $4, w = $5, dink = $6, dart = $7, raw_log = $10;`, - headerID, vatTune.Ilk, vatTune.Urn, vatTune.V, vatTune.W, vatTune.Dink, vatTune.Dart, vatTune.TransactionIndex, vatTune.LogIndex, vatTune.Raw, + headerID, ilkID, vatTune.Urn, vatTune.V, vatTune.W, vatTune.Dink, vatTune.Dart, vatTune.TransactionIndex, vatTune.LogIndex, vatTune.Raw, ) if execErr != nil { rollbackErr := tx.Rollback() diff --git a/pkg/transformers/vat_tune/repository_test.go b/pkg/transformers/vat_tune/repository_test.go index 1700825b..fe047c03 100644 --- a/pkg/transformers/vat_tune/repository_test.go +++ b/pkg/transformers/vat_tune/repository_test.go @@ -1,8 +1,10 @@ package vat_tune_test import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/shared_behaviors" + "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -51,7 +53,9 @@ var _ = Describe("Vat tune repository", func() { var dbVatTune vat_tune.VatTuneModel err = db.Get(&dbVatTune, `SELECT ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log FROM maker.vat_tune WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbVatTune.Ilk).To(Equal(test_data.VatTuneModel.Ilk)) + ilkID, err := shared.GetOrCreateIlk(test_data.VatTuneModel.Ilk, db) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatTune.Ilk).To(Equal(strconv.Itoa(ilkID))) Expect(dbVatTune.Urn).To(Equal(test_data.VatTuneModel.Urn)) Expect(dbVatTune.V).To(Equal(test_data.VatTuneModel.V)) Expect(dbVatTune.W).To(Equal(test_data.VatTuneModel.W)) diff --git a/test_config/test_config.go b/test_config/test_config.go index b760be06..ac954100 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -113,19 +113,36 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.flip_kick") db.MustExec("DELETE FROM maker.flop_kick") db.MustExec("DELETE FROM maker.frob") + db.MustExec("DELETE FROM maker.pit_drip") db.MustExec("DELETE FROM maker.pit_file_debt_ceiling") db.MustExec("DELETE FROM maker.pit_file_ilk") + db.MustExec("DELETE FROM maker.pit_ilk_line") + db.MustExec("DELETE FROM maker.pit_ilk_spot") + db.MustExec("DELETE FROM maker.pit_line") + db.MustExec("DELETE FROM maker.pit_live") + db.MustExec("DELETE FROM maker.pit_vat") db.MustExec("DELETE FROM maker.price_feeds") db.MustExec("DELETE FROM maker.tend") + db.MustExec("DELETE FROM maker.vat_dai") + db.MustExec("DELETE FROM maker.vat_debt") db.MustExec("DELETE FROM maker.vat_flux") db.MustExec("DELETE FROM maker.vat_fold") + db.MustExec("DELETE FROM maker.vat_gem") db.MustExec("DELETE FROM maker.vat_grab") db.MustExec("DELETE FROM maker.vat_heal") + db.MustExec("DELETE FROM maker.vat_ilk_art") + db.MustExec("DELETE FROM maker.vat_ilk_ink") + db.MustExec("DELETE FROM maker.vat_ilk_rate") + db.MustExec("DELETE FROM maker.vat_ilk_take") db.MustExec("DELETE FROM maker.vat_init") db.MustExec("DELETE FROM maker.vat_move") + db.MustExec("DELETE FROM maker.vat_sin") db.MustExec("DELETE FROM maker.vat_slip") db.MustExec("DELETE FROM maker.vat_toll") db.MustExec("DELETE FROM maker.vat_tune") + db.MustExec("DELETE FROM maker.vat_urn_art") + db.MustExec("DELETE FROM maker.vat_urn_ink") + db.MustExec("DELETE FROM maker.vat_vice") db.MustExec("DELETE FROM maker.vow_flog") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") @@ -141,6 +158,8 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.cat_flip_ilk") db.MustExec("DELETE FROM maker.cat_flip_ink") db.MustExec("DELETE FROM maker.cat_flip_tab") + // TODO: add ON DELETE CASCADE? otherwise this needs to come after deleting tables that reference it + db.MustExec("DELETE FROM maker.ilks") } // Returns a new test node, with the same ID