From 750754caa75d8c12a614a7dfa3b5404e153295a0 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 1 Aug 2019 10:33:41 -0500 Subject: [PATCH 01/13] Add an addresses table --- db/migrations/00028_add_address_table.sql | 11 +++++ db/schema.sql | 53 +++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 db/migrations/00028_add_address_table.sql diff --git a/db/migrations/00028_add_address_table.sql b/db/migrations/00028_add_address_table.sql new file mode 100644 index 00000000..2a7f99a8 --- /dev/null +++ b/db/migrations/00028_add_address_table.sql @@ -0,0 +1,11 @@ +-- +goose Up +CREATE TABLE public.addresses +( + id SERIAL PRIMARY KEY, + address character varying(42), + UNIQUE (address) +); + +-- +goose Down +-- SQL in this section is executed when the migration is rolled back. +DROP TABLE public.addresses; diff --git a/db/schema.sql b/db/schema.sql index a9fd68af..54f51241 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -20,6 +20,36 @@ SET default_tablespace = ''; SET default_with_oids = false; +-- +-- Name: addresses; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.addresses ( + id integer NOT NULL, + address character varying(42) +); + + +-- +-- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.addresses_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id; + + -- -- Name: logs; Type: TABLE; Schema: public; Owner: - -- @@ -564,6 +594,13 @@ CREATE VIEW public.watched_event_logs AS WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL))); +-- +-- Name: addresses id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass); + + -- -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -662,6 +699,22 @@ ALTER TABLE ONLY public.uncles ALTER COLUMN id SET DEFAULT nextval('public.uncle ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass); +-- +-- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_address_key UNIQUE (address); + + +-- +-- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_pkey PRIMARY KEY (id); + + -- -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- From 3005a1ad50e97ca0ee4b0ed2143bd96f909db700 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 1 Aug 2019 11:54:56 -0500 Subject: [PATCH 02/13] Move add address migration table up in the migration order --- ...00028_add_address_table.sql => 00013_add_address_table.sql} | 3 +-- ...eate_receipts_table.sql => 00014_create_receipts_table.sql} | 0 ...ipts.sql => 00015_add_transaction_id_index_to_receipts.sql} | 0 ...ceipts_fk_to_logs.sql => 00016_add_receipts_fk_to_logs.sql} | 0 ...016_create_log_filters.sql => 00017_create_log_filters.sql} | 0 ...ched_event_logs.sql => 00018_create_watched_event_logs.sql} | 0 ...nt.sql => 00019_update_log_filters_to_block_constraint.sql} | 0 ...00019_rename_node_table.sql => 00020_rename_node_table.sql} | 0 ...ith_blocks.sql => 00021_associate_receipts_with_blocks.sql} | 0 ...blocks.sql => 00022_add_eth_node_fingerprint_to_blocks.sql} | 0 ...create_headers_table.sql => 00023_create_headers_table.sql} | 0 ...eaders_table.sql => 00024_create_checked_headers_table.sql} | 0 ...eate_queued_storage.sql => 00025_create_queued_storage.sql} | 0 ...ble.sql => 00026_create_header_sync_transactions_table.sql} | 0 ...s_table.sql => 00027_create_header_sync_receipts_table.sql} | 0 ...7_create_uncles_table.sql => 00028_create_uncles_table.sql} | 0 16 files changed, 1 insertion(+), 2 deletions(-) rename db/migrations/{00028_add_address_table.sql => 00013_add_address_table.sql} (60%) rename db/migrations/{00013_create_receipts_table.sql => 00014_create_receipts_table.sql} (100%) rename db/migrations/{00014_add_transaction_id_index_to_receipts.sql => 00015_add_transaction_id_index_to_receipts.sql} (100%) rename db/migrations/{00015_add_receipts_fk_to_logs.sql => 00016_add_receipts_fk_to_logs.sql} (100%) rename db/migrations/{00016_create_log_filters.sql => 00017_create_log_filters.sql} (100%) rename db/migrations/{00017_create_watched_event_logs.sql => 00018_create_watched_event_logs.sql} (100%) rename db/migrations/{00018_update_log_filters_to_block_constraint.sql => 00019_update_log_filters_to_block_constraint.sql} (100%) rename db/migrations/{00019_rename_node_table.sql => 00020_rename_node_table.sql} (100%) rename db/migrations/{00020_associate_receipts_with_blocks.sql => 00021_associate_receipts_with_blocks.sql} (100%) rename db/migrations/{00021_add_eth_node_fingerprint_to_blocks.sql => 00022_add_eth_node_fingerprint_to_blocks.sql} (100%) rename db/migrations/{00022_create_headers_table.sql => 00023_create_headers_table.sql} (100%) rename db/migrations/{00023_create_checked_headers_table.sql => 00024_create_checked_headers_table.sql} (100%) rename db/migrations/{00024_create_queued_storage.sql => 00025_create_queued_storage.sql} (100%) rename db/migrations/{00025_create_header_sync_transactions_table.sql => 00026_create_header_sync_transactions_table.sql} (100%) rename db/migrations/{00026_create_header_sync_receipts_table.sql => 00027_create_header_sync_receipts_table.sql} (100%) rename db/migrations/{00027_create_uncles_table.sql => 00028_create_uncles_table.sql} (100%) diff --git a/db/migrations/00028_add_address_table.sql b/db/migrations/00013_add_address_table.sql similarity index 60% rename from db/migrations/00028_add_address_table.sql rename to db/migrations/00013_add_address_table.sql index 2a7f99a8..22ee7b3e 100644 --- a/db/migrations/00028_add_address_table.sql +++ b/db/migrations/00013_add_address_table.sql @@ -7,5 +7,4 @@ CREATE TABLE public.addresses ); -- +goose Down --- SQL in this section is executed when the migration is rolled back. -DROP TABLE public.addresses; +DROP TABLE public.addresses; \ No newline at end of file diff --git a/db/migrations/00013_create_receipts_table.sql b/db/migrations/00014_create_receipts_table.sql similarity index 100% rename from db/migrations/00013_create_receipts_table.sql rename to db/migrations/00014_create_receipts_table.sql diff --git a/db/migrations/00014_add_transaction_id_index_to_receipts.sql b/db/migrations/00015_add_transaction_id_index_to_receipts.sql similarity index 100% rename from db/migrations/00014_add_transaction_id_index_to_receipts.sql rename to db/migrations/00015_add_transaction_id_index_to_receipts.sql diff --git a/db/migrations/00015_add_receipts_fk_to_logs.sql b/db/migrations/00016_add_receipts_fk_to_logs.sql similarity index 100% rename from db/migrations/00015_add_receipts_fk_to_logs.sql rename to db/migrations/00016_add_receipts_fk_to_logs.sql diff --git a/db/migrations/00016_create_log_filters.sql b/db/migrations/00017_create_log_filters.sql similarity index 100% rename from db/migrations/00016_create_log_filters.sql rename to db/migrations/00017_create_log_filters.sql diff --git a/db/migrations/00017_create_watched_event_logs.sql b/db/migrations/00018_create_watched_event_logs.sql similarity index 100% rename from db/migrations/00017_create_watched_event_logs.sql rename to db/migrations/00018_create_watched_event_logs.sql diff --git a/db/migrations/00018_update_log_filters_to_block_constraint.sql b/db/migrations/00019_update_log_filters_to_block_constraint.sql similarity index 100% rename from db/migrations/00018_update_log_filters_to_block_constraint.sql rename to db/migrations/00019_update_log_filters_to_block_constraint.sql diff --git a/db/migrations/00019_rename_node_table.sql b/db/migrations/00020_rename_node_table.sql similarity index 100% rename from db/migrations/00019_rename_node_table.sql rename to db/migrations/00020_rename_node_table.sql diff --git a/db/migrations/00020_associate_receipts_with_blocks.sql b/db/migrations/00021_associate_receipts_with_blocks.sql similarity index 100% rename from db/migrations/00020_associate_receipts_with_blocks.sql rename to db/migrations/00021_associate_receipts_with_blocks.sql diff --git a/db/migrations/00021_add_eth_node_fingerprint_to_blocks.sql b/db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql similarity index 100% rename from db/migrations/00021_add_eth_node_fingerprint_to_blocks.sql rename to db/migrations/00022_add_eth_node_fingerprint_to_blocks.sql diff --git a/db/migrations/00022_create_headers_table.sql b/db/migrations/00023_create_headers_table.sql similarity index 100% rename from db/migrations/00022_create_headers_table.sql rename to db/migrations/00023_create_headers_table.sql diff --git a/db/migrations/00023_create_checked_headers_table.sql b/db/migrations/00024_create_checked_headers_table.sql similarity index 100% rename from db/migrations/00023_create_checked_headers_table.sql rename to db/migrations/00024_create_checked_headers_table.sql diff --git a/db/migrations/00024_create_queued_storage.sql b/db/migrations/00025_create_queued_storage.sql similarity index 100% rename from db/migrations/00024_create_queued_storage.sql rename to db/migrations/00025_create_queued_storage.sql diff --git a/db/migrations/00025_create_header_sync_transactions_table.sql b/db/migrations/00026_create_header_sync_transactions_table.sql similarity index 100% rename from db/migrations/00025_create_header_sync_transactions_table.sql rename to db/migrations/00026_create_header_sync_transactions_table.sql diff --git a/db/migrations/00026_create_header_sync_receipts_table.sql b/db/migrations/00027_create_header_sync_receipts_table.sql similarity index 100% rename from db/migrations/00026_create_header_sync_receipts_table.sql rename to db/migrations/00027_create_header_sync_receipts_table.sql diff --git a/db/migrations/00027_create_uncles_table.sql b/db/migrations/00028_create_uncles_table.sql similarity index 100% rename from db/migrations/00027_create_uncles_table.sql rename to db/migrations/00028_create_uncles_table.sql From b13eefbce720bb2f9a1d69937d90afdc69e58137 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 1 Aug 2019 15:24:02 -0500 Subject: [PATCH 03/13] Add AddressRepository --- .../repositories/address_repository.go | 40 +++++++++ .../repositories/address_repository_test.go | 89 +++++++++++++++++++ pkg/datastore/repository.go | 4 + 3 files changed, 133 insertions(+) create mode 100644 pkg/datastore/postgres/repositories/address_repository.go create mode 100644 pkg/datastore/postgres/repositories/address_repository_test.go diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go new file mode 100644 index 00000000..9927d099 --- /dev/null +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -0,0 +1,40 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repositories + +import ( + "database/sql" + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type AddressRepository struct { + *postgres.DB +} + +func (repo AddressRepository) CreateOrGetAddress(address string) (int, error) { + stringAddressToCommonAddress := common.HexToAddress(address) + hexAddress := stringAddressToCommonAddress.Hex() + + var addressId int + getErr := repo.DB.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) + if getErr == sql.ErrNoRows { + insertErr := repo.DB.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) + return addressId, insertErr + } + + return addressId, getErr +} + diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go new file mode 100644 index 00000000..417bc9ab --- /dev/null +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -0,0 +1,89 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repositories_test + +import ( + . "github.com/onsi/ginkgo" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/test_config" + "strings" + + . "github.com/onsi/gomega" +) + +var _ = Describe("address repository", func() { + var ( + db *postgres.DB + repo repositories.AddressRepository + address = fakes.FakeAddress.Hex() + ) + BeforeEach(func() { + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + repo = repositories.AddressRepository{DB: db} + }) + + type dbAddress struct { + Id int + Address string + } + + It("creates an address record", func() { + addressId, createErr := repo.CreateOrGetAddress(address) + Expect(createErr).NotTo(HaveOccurred()) + + var actualAddress dbAddress + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) + Expect(getErr).NotTo(HaveOccurred()) + expectedAddress := dbAddress{Id: addressId, Address: address} + Expect(actualAddress).To(Equal(expectedAddress)) + }) + + It("returns the existing record id if the address already exists", func() { + _, createErr := repo.CreateOrGetAddress(address) + Expect(createErr).NotTo(HaveOccurred()) + + _, getErr := repo.CreateOrGetAddress(address) + Expect(getErr).NotTo(HaveOccurred()) + + var addressCount int + addressErr := repo.DB.Get(&addressCount, `SELECT count(*) FROM public.addresses`) + Expect(addressErr).NotTo(HaveOccurred()) + }) + + It("gets upper-cased addresses", func() { + //insert it as all upper + upperAddress := strings.ToUpper(address) + upperAddressId, createErr := repo.CreateOrGetAddress(upperAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.CreateOrGetAddress(address) + Expect(getErr).NotTo(HaveOccurred()) + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) + + It("gets lower-cased addresses", func() { + //insert it as all upper + lowerAddress := strings.ToLower(address) + upperAddressId, createErr := repo.CreateOrGetAddress(lowerAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.CreateOrGetAddress(address) + Expect(getErr).NotTo(HaveOccurred()) + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) +}) \ No newline at end of file diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index db0c220b..ffcebfa8 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -21,6 +21,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/filters" ) +type AddressRepository interface { + GetOrCreateAddress(address string) (int, error) +} + type BlockRepository interface { CreateOrUpdateBlock(block core.Block) (int64, error) GetBlock(blockNumber int64) (core.Block, error) From 1373fe83a1a5d0293b4eb879dae4fa1062a5dd98 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 1 Aug 2019 15:44:13 -0500 Subject: [PATCH 04/13] Pass a db into GetOrCreateAddress --- .../repositories/address_repository.go | 10 ++++------ .../repositories/address_repository_test.go | 18 +++++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 9927d099..ff554112 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -20,18 +20,16 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type AddressRepository struct { - *postgres.DB -} +type AddressRepository struct {} -func (repo AddressRepository) CreateOrGetAddress(address string) (int, error) { +func (repo AddressRepository) CreateOrGetAddress(db *postgres.DB, address string) (int, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() var addressId int - getErr := repo.DB.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) + getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) if getErr == sql.ErrNoRows { - insertErr := repo.DB.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) + insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) return addressId, insertErr } diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index 417bc9ab..db13673e 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -34,7 +34,7 @@ var _ = Describe("address repository", func() { BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - repo = repositories.AddressRepository{DB: db} + repo = repositories.AddressRepository{} }) type dbAddress struct { @@ -43,7 +43,7 @@ var _ = Describe("address repository", func() { } It("creates an address record", func() { - addressId, createErr := repo.CreateOrGetAddress(address) + addressId, createErr := repo.CreateOrGetAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) var actualAddress dbAddress @@ -54,24 +54,24 @@ var _ = Describe("address repository", func() { }) It("returns the existing record id if the address already exists", func() { - _, createErr := repo.CreateOrGetAddress(address) + _, createErr := repo.CreateOrGetAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) - _, getErr := repo.CreateOrGetAddress(address) + _, getErr := repo.CreateOrGetAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) var addressCount int - addressErr := repo.DB.Get(&addressCount, `SELECT count(*) FROM public.addresses`) + addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) Expect(addressErr).NotTo(HaveOccurred()) }) It("gets upper-cased addresses", func() { //insert it as all upper upperAddress := strings.ToUpper(address) - upperAddressId, createErr := repo.CreateOrGetAddress(upperAddress) + upperAddressId, createErr := repo.CreateOrGetAddress(db, upperAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.CreateOrGetAddress(address) + mixedCaseAddressId, getErr := repo.CreateOrGetAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) Expect(upperAddressId).To(Equal(mixedCaseAddressId)) }) @@ -79,10 +79,10 @@ var _ = Describe("address repository", func() { It("gets lower-cased addresses", func() { //insert it as all upper lowerAddress := strings.ToLower(address) - upperAddressId, createErr := repo.CreateOrGetAddress(lowerAddress) + upperAddressId, createErr := repo.CreateOrGetAddress(db, lowerAddress) Expect(createErr).NotTo(HaveOccurred()) - mixedCaseAddressId, getErr := repo.CreateOrGetAddress(address) + mixedCaseAddressId, getErr := repo.CreateOrGetAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) Expect(upperAddressId).To(Equal(mixedCaseAddressId)) }) From 4b61c87b553421b4ff5c1962d5fec7aa9b5be0de Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 2 Aug 2019 08:52:14 -0500 Subject: [PATCH 05/13] Get or create address record in a transaction --- .../repositories/address_repository.go | 18 ++- .../repositories/address_repository_test.go | 150 ++++++++++++------ 2 files changed, 121 insertions(+), 47 deletions(-) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index ff554112..7e9d4724 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -17,12 +17,13 @@ package repositories import ( "database/sql" "github.com/ethereum/go-ethereum/common" + "github.com/jmoiron/sqlx" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type AddressRepository struct {} +type AddressRepository struct{} -func (repo AddressRepository) CreateOrGetAddress(db *postgres.DB, address string) (int, error) { +func (repo AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() @@ -36,3 +37,16 @@ func (repo AddressRepository) CreateOrGetAddress(db *postgres.DB, address string return addressId, getErr } +func (repo AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int, error) { + stringAddressToCommonAddress := common.HexToAddress(address) + hexAddress := stringAddressToCommonAddress.Hex() + + var addressId int + getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) + if getErr == sql.ErrNoRows { + insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) + return addressId, insertErr + } + + return addressId, getErr +} diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index db13673e..b370ea18 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -15,20 +15,20 @@ package repositories_test import ( + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/test_config" "strings" - - . "github.com/onsi/gomega" ) -var _ = Describe("address repository", func() { +var _ = Describe("address lookup", func() { var ( - db *postgres.DB - repo repositories.AddressRepository + db *postgres.DB + repo repositories.AddressRepository address = fakes.FakeAddress.Hex() ) BeforeEach(func() { @@ -38,52 +38,112 @@ var _ = Describe("address repository", func() { }) type dbAddress struct { - Id int + Id int Address string } - It("creates an address record", func() { - addressId, createErr := repo.CreateOrGetAddress(db, address) - Expect(createErr).NotTo(HaveOccurred()) + Describe("GetOrCreateAddress", func() { + It("creates an address record", func() { + addressId, createErr := repo.GetOrCreateAddress(db, address) + Expect(createErr).NotTo(HaveOccurred()) - var actualAddress dbAddress - getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) - Expect(getErr).NotTo(HaveOccurred()) - expectedAddress := dbAddress{Id: addressId, Address: address} - Expect(actualAddress).To(Equal(expectedAddress)) + var actualAddress dbAddress + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) + Expect(getErr).NotTo(HaveOccurred()) + expectedAddress := dbAddress{Id: addressId, Address: address} + Expect(actualAddress).To(Equal(expectedAddress)) + }) + + It("returns the existing record id if the address already exists", func() { + _, createErr := repo.GetOrCreateAddress(db, address) + Expect(createErr).NotTo(HaveOccurred()) + + _, getErr := repo.GetOrCreateAddress(db, address) + Expect(getErr).NotTo(HaveOccurred()) + + var addressCount int + addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) + Expect(addressErr).NotTo(HaveOccurred()) + }) + + It("gets upper-cased addresses", func() { + upperAddress := strings.ToUpper(address) + upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address) + Expect(getErr).NotTo(HaveOccurred()) + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) + + It("gets lower-cased addresses", func() { + lowerAddress := strings.ToLower(address) + upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address) + Expect(getErr).NotTo(HaveOccurred()) + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) }) - It("returns the existing record id if the address already exists", func() { - _, createErr := repo.CreateOrGetAddress(db, address) - Expect(createErr).NotTo(HaveOccurred()) + Describe("GetOrCreateAddressInTransaction", func() { + var ( + tx *sqlx.Tx + txErr error + ) + BeforeEach(func() { + tx, txErr = db.Beginx() + Expect(txErr).NotTo(HaveOccurred()) + }) - _, getErr := repo.CreateOrGetAddress(db, address) - Expect(getErr).NotTo(HaveOccurred()) + It("creates an address record", func() { + addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address) + Expect(createErr).NotTo(HaveOccurred()) + tx.Commit() - var addressCount int - addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) - Expect(addressErr).NotTo(HaveOccurred()) + var actualAddress dbAddress + getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`) + Expect(getErr).NotTo(HaveOccurred()) + expectedAddress := dbAddress{Id: addressId, Address: address} + Expect(actualAddress).To(Equal(expectedAddress)) + }) + + It("returns the existing record id if the address already exists", func() { + _, createErr := repo.GetOrCreateAddressInTransaction(tx, address) + Expect(createErr).NotTo(HaveOccurred()) + + _, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + Expect(getErr).NotTo(HaveOccurred()) + tx.Commit() + + var addressCount int + addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) + Expect(addressErr).NotTo(HaveOccurred()) + }) + + It("gets upper-cased addresses", func() { + upperAddress := strings.ToUpper(address) + upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + Expect(getErr).NotTo(HaveOccurred()) + tx.Commit() + + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) + + It("gets lower-cased addresses", func() { + lowerAddress := strings.ToLower(address) + upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress) + Expect(createErr).NotTo(HaveOccurred()) + + mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address) + Expect(getErr).NotTo(HaveOccurred()) + tx.Commit() + + Expect(upperAddressId).To(Equal(mixedCaseAddressId)) + }) }) - - It("gets upper-cased addresses", func() { - //insert it as all upper - upperAddress := strings.ToUpper(address) - upperAddressId, createErr := repo.CreateOrGetAddress(db, upperAddress) - Expect(createErr).NotTo(HaveOccurred()) - - mixedCaseAddressId, getErr := repo.CreateOrGetAddress(db, address) - Expect(getErr).NotTo(HaveOccurred()) - Expect(upperAddressId).To(Equal(mixedCaseAddressId)) - }) - - It("gets lower-cased addresses", func() { - //insert it as all upper - lowerAddress := strings.ToLower(address) - upperAddressId, createErr := repo.CreateOrGetAddress(db, lowerAddress) - Expect(createErr).NotTo(HaveOccurred()) - - mixedCaseAddressId, getErr := repo.CreateOrGetAddress(db, address) - Expect(getErr).NotTo(HaveOccurred()) - Expect(upperAddressId).To(Equal(mixedCaseAddressId)) - }) -}) \ No newline at end of file +}) From 258035833b2541f8c403fe3d5d09da72b352c975 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 2 Aug 2019 09:58:30 -0500 Subject: [PATCH 06/13] Add GetAddressById --- .../repositories/address_repository.go | 9 +++++++++ .../repositories/address_repository_test.go | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 7e9d4724..950fe50c 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -50,3 +50,12 @@ func (repo AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, addre return addressId, getErr } + +func (repo AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error){ + var address string + getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) + if getErr != nil { + return "", getErr + } + return address, nil +} diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index b370ea18..c0f3fa82 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -146,4 +146,22 @@ var _ = Describe("address lookup", func() { Expect(upperAddressId).To(Equal(mixedCaseAddressId)) }) }) + + Describe("GetAddressById", func() { + It("gets and address by it's id", func() { + addressId, createErr := repo.GetOrCreateAddress(db, address) + Expect(createErr).NotTo(HaveOccurred()) + + actualAddress, getErr := repo.GetAddressById(db, addressId) + Expect(getErr).NotTo(HaveOccurred()) + Expect(actualAddress).To(Equal(address)) + }) + + It("returns an error if the id doesn't exist", func() { + _, getErr := repo.GetAddressById(db, 0) + Expect(getErr).To(HaveOccurred()) + Expect(getErr).To(MatchError("sql: no rows in result set")) + }) + }) }) + From bcd6d14fcdef17dbb9d7fc48fd10af218ea7bdf1 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 2 Aug 2019 10:09:59 -0500 Subject: [PATCH 07/13] Updating full_sync_receipts to have FK reference to addresses --- db/migrations/00014_create_receipts_table.sql | 16 ++++----- db/schema.sql | 10 +++++- .../full/retriever/block_retriever.go | 23 ++++++++++--- .../shared/helpers/test_helpers/database.go | 3 ++ .../repositories/address_repository.go | 2 +- .../repositories/address_repository_test.go | 1 - .../postgres/repositories/block_repository.go | 26 +++++---------- .../repositories/logs_repository_test.go | 8 +++-- .../repositories/receipt_repository.go | 33 +++++++------------ .../watched_events_repository_test.go | 8 +++-- pkg/datastore/repository.go | 3 +- pkg/fakes/mock_receipt_repository.go | 3 +- test_config/test_config.go | 1 + 13 files changed, 75 insertions(+), 62 deletions(-) diff --git a/db/migrations/00014_create_receipts_table.sql b/db/migrations/00014_create_receipts_table.sql index 2864f226..b39e4492 100644 --- a/db/migrations/00014_create_receipts_table.sql +++ b/db/migrations/00014_create_receipts_table.sql @@ -1,14 +1,14 @@ -- +goose Up CREATE TABLE full_sync_receipts ( - id SERIAL PRIMARY KEY, - transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE, - contract_address VARCHAR(42), - cumulative_gas_used NUMERIC, - gas_used NUMERIC, - state_root VARCHAR(66), - status INTEGER, - tx_hash VARCHAR(66) + id SERIAL PRIMARY KEY, + transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE, + contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE, + cumulative_gas_used NUMERIC, + gas_used NUMERIC, + state_root VARCHAR(66), + status INTEGER, + tx_hash VARCHAR(66) ); diff --git a/db/schema.sql b/db/schema.sql index 54f51241..45801bea 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -174,7 +174,7 @@ CREATE TABLE public.eth_nodes ( CREATE TABLE public.full_sync_receipts ( id integer NOT NULL, - contract_address character varying(42), + contract_address_id integer NOT NULL, cumulative_gas_used numeric, gas_used numeric, state_root character varying(66), @@ -949,6 +949,14 @@ ALTER TABLE ONLY public.checked_headers ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: full_sync_receipts full_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.full_sync_receipts + ADD CONSTRAINT full_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE; + + -- -- Name: full_sync_transactions full_sync_transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/contract_watcher/full/retriever/block_retriever.go b/pkg/contract_watcher/full/retriever/block_retriever.go index 2f595cb2..80f330ad 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever.go +++ b/pkg/contract_watcher/full/retriever/block_retriever.go @@ -17,7 +17,9 @@ package retriever import ( + "database/sql" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) // Block retriever is used to retrieve the first block for a given contract and the most recent block @@ -41,7 +43,10 @@ func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) { i, err := r.retrieveFirstBlockFromReceipts(contractAddr) if err != nil { - i, err = r.retrieveFirstBlockFromLogs(contractAddr) + if err == sql.ErrNoRows { + i, err = r.retrieveFirstBlockFromLogs(contractAddr) + } + return i, err } return i, err @@ -49,18 +54,26 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { - var firstBlock int + var firstBlock int64 + addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr) + if getAddressErr != nil { + return firstBlock, getAddressErr + } err := r.db.Get( &firstBlock, `SELECT number FROM blocks WHERE id = (SELECT block_id FROM full_sync_receipts - WHERE lower(contract_address) = $1 + WHERE contract_address_id = $1 ORDER BY block_id ASC LIMIT 1)`, - contractAddr, + addressId, ) - return int64(firstBlock), err + return firstBlock, err +} + +func addressRepository() repositories.AddressRepository { + return repositories.AddressRepository{} } // In which case this servers as a heuristic to find the first block by finding the first contract event log diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index fb3f2e70..c274fda1 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -225,6 +225,9 @@ func TearDown(db *postgres.DB) { tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM addresses`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM blocks`) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 950fe50c..1b04681c 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -51,7 +51,7 @@ func (repo AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, addre return addressId, getErr } -func (repo AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error){ +func (repo AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error) { var address string getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) if getErr != nil { diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index c0f3fa82..3935b9b5 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -164,4 +164,3 @@ var _ = Describe("address lookup", func() { }) }) }) - diff --git a/pkg/datastore/postgres/repositories/block_repository.go b/pkg/datastore/postgres/repositories/block_repository.go index 979bd633..db2af9c8 100644 --- a/pkg/datastore/postgres/repositories/block_repository.go +++ b/pkg/datastore/postgres/repositories/block_repository.go @@ -234,7 +234,8 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in return err } if hasReceipt(transaction) { - receiptId, err := blockRepository.createReceipt(tx, blockId, transaction.Receipt) + + receiptId, err := receiptRepository().CreateReceipt(blockId, transaction.Receipt, tx) if err != nil { return err } @@ -248,6 +249,11 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in return nil } +func receiptRepository() datastore.ReceiptRepository { + //TODO: set db? + return ReceiptRepository{} +} + func hasLogs(transaction core.TransactionModel) bool { return len(transaction.Receipt.Logs) > 0 } @@ -256,22 +262,6 @@ func hasReceipt(transaction core.TransactionModel) bool { return transaction.Receipt.TxHash != "" } -func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) { - //Not currently persisting log bloom filters - var receiptId int - err := tx.QueryRow( - `INSERT INTO full_sync_receipts - (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id) - VALUES ($1, $2, $3, $4, $5, $6, $7) - RETURNING id`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId) - if err != nil { - log.Error("createReceipt: error inserting receipt: ", err) - return receiptId, err - } - return receiptId, nil -} - func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) { var retrievedBlockHash string // TODO: handle possible error @@ -283,7 +273,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b return retrievedBlockHash, blockExists(retrievedBlockHash) } -func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error { +func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int64) error { for _, tlog := range logs { _, err := tx.Exec( `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) diff --git a/pkg/datastore/postgres/repositories/logs_repository_test.go b/pkg/datastore/postgres/repositories/logs_repository_test.go index fe5d069d..f2f4c27b 100644 --- a/pkg/datastore/postgres/repositories/logs_repository_test.go +++ b/pkg/datastore/postgres/repositories/logs_repository_test.go @@ -55,7 +55,9 @@ var _ = Describe("Logs Repository", func() { blockNumber := int64(12345) blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) Expect(err).NotTo(HaveOccurred()) - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}) + tx, _ := db.Beginx() + receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logsRepository.CreateLogs([]core.Log{{ BlockNumber: blockNumber, @@ -91,7 +93,9 @@ var _ = Describe("Logs Repository", func() { blockNumber := int64(12345) blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) Expect(err).NotTo(HaveOccurred()) - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}) + tx, _ := db.Beginx() + receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logsRepository.CreateLogs([]core.Log{{ diff --git a/pkg/datastore/postgres/repositories/receipt_repository.go b/pkg/datastore/postgres/repositories/receipt_repository.go index 01e38f09..59099578 100644 --- a/pkg/datastore/postgres/repositories/receipt_repository.go +++ b/pkg/datastore/postgres/repositories/receipt_repository.go @@ -36,7 +36,7 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, return err } for _, receipt := range receipts { - receiptId, err := createReceipt(receipt, blockId, tx) + receiptId, err := receiptRepository.CreateReceipt(blockId, receipt, tx) if err != nil { tx.Rollback() return err @@ -53,21 +53,6 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, return nil } -func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) { - var receiptId int64 - err := tx.QueryRow( - `INSERT INTO full_sync_receipts - (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id) - VALUES ($1, $2, $3, $4, $5, $6, $7) - RETURNING id`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId, - ).Scan(&receiptId) - if err != nil { - logrus.Error("createReceipt: Error inserting: ", err) - } - return receiptId, err -} - func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error { for _, log := range logs { _, err := tx.Exec( @@ -83,27 +68,31 @@ func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error { return nil } -func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) { - tx, _ := receiptRepository.DB.Beginx() +//TODO: test that creating the address should be in the transaction +func (ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { var receiptId int64 + addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) + if getAddressErr != nil { + logrus.Error("createReceipt: Error getting address id: ", getAddressErr) + return receiptId, getAddressErr + } err := tx.QueryRow( `INSERT INTO full_sync_receipts - (contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id) + (contract_address_id, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, - receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId) + addressId, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId) if err != nil { tx.Rollback() logrus.Warning("CreateReceipt: error inserting receipt: ", err) return receiptId, err } - tx.Commit() return receiptId, nil } func (receiptRepository ReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) { row := receiptRepository.DB.QueryRow( - `SELECT contract_address, + `SELECT contract_address_id, tx_hash, cumulative_gas_used, gas_used, diff --git a/pkg/datastore/postgres/repositories/watched_events_repository_test.go b/pkg/datastore/postgres/repositories/watched_events_repository_test.go index 957d3c33..f30a85f5 100644 --- a/pkg/datastore/postgres/repositories/watched_events_repository_test.go +++ b/pkg/datastore/postgres/repositories/watched_events_repository_test.go @@ -79,7 +79,9 @@ var _ = Describe("Watched Events Repository", func() { Expect(err).ToNot(HaveOccurred()) blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{}) Expect(err).NotTo(HaveOccurred()) - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}) + tx, _ := db.Beginx() + receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logRepository.CreateLogs(logs, receiptId) Expect(err).ToNot(HaveOccurred()) @@ -136,7 +138,9 @@ var _ = Describe("Watched Events Repository", func() { Expect(err).ToNot(HaveOccurred()) blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{Hash: "Ox123"}) Expect(err).NotTo(HaveOccurred()) - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{TxHash: "0x123"}) + tx, _ := db.Beginx() + receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logRepository.CreateLogs(logs, receiptId) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index ffcebfa8..c1883ecc 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -17,6 +17,7 @@ package datastore import ( + "github.com/jmoiron/sqlx" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" ) @@ -57,7 +58,7 @@ type LogRepository interface { type ReceiptRepository interface { CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error - CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) + CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) //TODO: change the name to CreateReceiptInTransaction GetReceipt(txHash string) (core.Receipt, error) } diff --git a/pkg/fakes/mock_receipt_repository.go b/pkg/fakes/mock_receipt_repository.go index 63a0d24d..3aad1a57 100644 --- a/pkg/fakes/mock_receipt_repository.go +++ b/pkg/fakes/mock_receipt_repository.go @@ -17,6 +17,7 @@ package fakes import ( + "github.com/jmoiron/sqlx" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -49,7 +50,7 @@ func (mrr *MockReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts return mrr.createReceiptsAndLogsReturnErr } -func (mrr *MockReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) { +func (mrr *MockReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { panic("implement me") } diff --git a/test_config/test_config.go b/test_config/test_config.go index ac8acff1..33f6b5bc 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -105,6 +105,7 @@ func NewTestDB(node core.Node) *postgres.DB { } func CleanTestDB(db *postgres.DB) { + db.MustExec("DELETE FROM addresses") db.MustExec("DELETE FROM blocks") db.MustExec("DELETE FROM checked_headers") // can't delete from eth_nodes since this function is called after the required eth_node is persisted From 4e40e892d217a250b7f245580bb1741eb2cadb04 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 5 Aug 2019 09:04:29 -0500 Subject: [PATCH 08/13] Updating header_sync_receipts to have FK reference to addresses --- ...0027_create_header_sync_receipts_table.sql | 25 ++++++++--------- db/schema.sql | 10 ++++++- .../repositories/address_repository_test.go | 1 + .../repositories/header_repository.go | 11 +++++--- .../repositories/header_repository_test.go | 27 ++++++++++++++----- 5 files changed, 51 insertions(+), 23 deletions(-) diff --git a/db/migrations/00027_create_header_sync_receipts_table.sql b/db/migrations/00027_create_header_sync_receipts_table.sql index ebd0a17b..842527f4 100644 --- a/db/migrations/00027_create_header_sync_receipts_table.sql +++ b/db/migrations/00027_create_header_sync_receipts_table.sql @@ -1,16 +1,17 @@ -- +goose Up -CREATE TABLE header_sync_receipts( - id SERIAL PRIMARY KEY, - transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE, - header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE, - contract_address VARCHAR(42), - cumulative_gas_used NUMERIC, - gas_used NUMERIC, - state_root VARCHAR(66), - status INTEGER, - tx_hash VARCHAR(66), - rlp BYTEA, - UNIQUE(header_id, transaction_id) +CREATE TABLE header_sync_receipts +( + id SERIAL PRIMARY KEY, + transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE, + cumulative_gas_used NUMERIC, + gas_used NUMERIC, + state_root VARCHAR(66), + status INTEGER, + tx_hash VARCHAR(66), + rlp BYTEA, + UNIQUE (header_id, transaction_id) ); diff --git a/db/schema.sql b/db/schema.sql index 45801bea..5470a107 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -284,7 +284,7 @@ CREATE TABLE public.header_sync_receipts ( id integer NOT NULL, transaction_id integer NOT NULL, header_id integer NOT NULL, - contract_address character varying(42), + contract_address_id integer NOT NULL, cumulative_gas_used numeric, gas_used numeric, state_root character varying(66), @@ -965,6 +965,14 @@ ALTER TABLE ONLY public.full_sync_transactions ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE; +-- +-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.header_sync_receipts + ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE; + + -- -- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index 3935b9b5..1ac53672 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -64,6 +64,7 @@ var _ = Describe("address lookup", func() { var addressCount int addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) Expect(addressErr).NotTo(HaveOccurred()) + Expect(addressCount).To(Equal(1)) }) It("gets upper-cased addresses", func() { diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 62ff7ecd..4429fe8d 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -87,13 +87,18 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) { var receiptId int64 + addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) + if getAddressErr != nil { + log.Error("createReceipt: Error getting address id: ", getAddressErr) + return receiptId, getAddressErr + } err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts - (header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) + (header_id, transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (header_id, transaction_id) DO UPDATE - SET (contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9) + SET (contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9) RETURNING id`, - headerID, transactionID, receipt.ContractAddress, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId) + headerID, transactionID, addressId, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId) if err != nil { log.Error("header_repository: error inserting receipt: ", err) return receiptId, err diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index 68d20aa2..bf9ea80c 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -217,25 +217,38 @@ var _ = Describe("Block header repository", func() { } _, receiptErr := repo.CreateReceiptInTx(tx, headerID, txId, receipt) + Expect(receiptErr).ToNot(HaveOccurred()) commitErr := tx.Commit() Expect(commitErr).ToNot(HaveOccurred()) - Expect(receiptErr).ToNot(HaveOccurred()) type idModel struct { - TransactionId int64 `db:"transaction_id"` - core.Receipt + TransactionId int64 `db:"transaction_id"` + ContractAddressId int64 `db:"contract_address_id"` + CumulativeGasUsed uint64 `db:"cumulative_gas_used"` + GasUsed uint64 `db:"gas_used"` + StateRoot string `db:"state_root"` + Status int + TxHash string `db:"tx_hash"` + Rlp []byte `db:"rlp"` } + + var addressId int64 + getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex()) + Expect(getAddressErr).NotTo(HaveOccurred()) + var dbReceipt idModel - err = db.Get(&dbReceipt, - `SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp + getReceiptErr := db.Get(&dbReceipt, + `SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp FROM public.header_sync_receipts WHERE header_id = $1`, headerID) - Expect(err).NotTo(HaveOccurred()) + Expect(getReceiptErr).NotTo(HaveOccurred()) + Expect(dbReceipt.TransactionId).To(Equal(txId)) Expect(dbReceipt.TxHash).To(Equal(txHash.Hex())) - Expect(dbReceipt.ContractAddress).To(Equal(contractAddr.Hex())) + Expect(dbReceipt.ContractAddressId).To(Equal(addressId)) Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100))) Expect(dbReceipt.GasUsed).To(Equal(uint64(10))) Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex())) + Expect(dbReceipt.Status).To(Equal(0)) Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3})) }) }) From 7e387646186a1fa759a2104232b922cb6ad615f1 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 5 Aug 2019 10:38:37 -0500 Subject: [PATCH 09/13] Rename ReceiptRepository -> FullSyncReceiptRepository --- cmd/coldImport.go | 2 +- .../shared/helpers/test_helpers/database.go | 4 ++-- .../postgres/repositories/address_repository.go | 6 +++--- .../postgres/repositories/block_repository.go | 9 ++------- ...pository.go => full_sync_receipt_repository.go} | 11 +++++------ ...est.go => full_sync_receipt_repository_test.go} | 14 +++++++------- .../postgres/repositories/logs_repository_test.go | 8 ++++---- .../repositories/watched_events_repository_test.go | 8 ++++---- pkg/datastore/repository.go | 6 +++--- pkg/fakes/mock_receipt_repository.go | 4 ++-- pkg/geth/cold_import/importer.go | 4 ++-- 11 files changed, 35 insertions(+), 41 deletions(-) rename pkg/datastore/postgres/repositories/{receipt_repository.go => full_sync_receipt_repository.go} (88%) rename pkg/datastore/postgres/repositories/{receipt_repository_test.go => full_sync_receipt_repository_test.go} (90%) diff --git a/cmd/coldImport.go b/cmd/coldImport.go index 134d47ca..ddd1048b 100644 --- a/cmd/coldImport.go +++ b/cmd/coldImport.go @@ -84,7 +84,7 @@ func coldImport() { // init cold importer deps blockRepository := repositories.NewBlockRepository(&pgDB) - receiptRepository := repositories.ReceiptRepository{DB: &pgDB} + receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB} transactionConverter := cold_db.NewColdDbTransactionConverter() blockConverter := vulcCommon.NewBlockConverter(transactionConverter) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index c274fda1..294a74c6 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -137,7 +137,7 @@ func SetupTusdRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) }, core.Node{}) Expect(err).NotTo(HaveOccurred()) - receiptRepository := repositories.ReceiptRepository{DB: db} + receiptRepository := repositories.FullSyncReceiptRepository{DB: db} logRepository := repositories.LogRepository{DB: db} blockRepository := *repositories.NewBlockRepository(db) @@ -183,7 +183,7 @@ func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) ( }, core.Node{}) Expect(err).NotTo(HaveOccurred()) - receiptRepository := repositories.ReceiptRepository{DB: db} + receiptRepository := repositories.FullSyncReceiptRepository{DB: db} logRepository := repositories.LogRepository{DB: db} blockRepository := *repositories.NewBlockRepository(db) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 1b04681c..da25bda6 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -23,7 +23,7 @@ import ( type AddressRepository struct{} -func (repo AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int, error) { +func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() @@ -37,7 +37,7 @@ func (repo AddressRepository) GetOrCreateAddress(db *postgres.DB, address string return addressId, getErr } -func (repo AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int, error) { +func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() @@ -51,7 +51,7 @@ func (repo AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, addre return addressId, getErr } -func (repo AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error) { +func (AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error) { var address string getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) if getErr != nil { diff --git a/pkg/datastore/postgres/repositories/block_repository.go b/pkg/datastore/postgres/repositories/block_repository.go index db2af9c8..2242efef 100644 --- a/pkg/datastore/postgres/repositories/block_repository.go +++ b/pkg/datastore/postgres/repositories/block_repository.go @@ -234,8 +234,8 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in return err } if hasReceipt(transaction) { - - receiptId, err := receiptRepository().CreateReceipt(blockId, transaction.Receipt, tx) + receiptRepo := FullSyncReceiptRepository{} + receiptId, err := receiptRepo.CreateFullSyncReceiptInTx(blockId, transaction.Receipt, tx) if err != nil { return err } @@ -249,11 +249,6 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in return nil } -func receiptRepository() datastore.ReceiptRepository { - //TODO: set db? - return ReceiptRepository{} -} - func hasLogs(transaction core.TransactionModel) bool { return len(transaction.Receipt.Logs) > 0 } diff --git a/pkg/datastore/postgres/repositories/receipt_repository.go b/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go similarity index 88% rename from pkg/datastore/postgres/repositories/receipt_repository.go rename to pkg/datastore/postgres/repositories/full_sync_receipt_repository.go index 59099578..71a345f6 100644 --- a/pkg/datastore/postgres/repositories/receipt_repository.go +++ b/pkg/datastore/postgres/repositories/full_sync_receipt_repository.go @@ -26,17 +26,17 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type ReceiptRepository struct { +type FullSyncReceiptRepository struct { *postgres.DB } -func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error { +func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error { tx, err := receiptRepository.DB.Beginx() if err != nil { return err } for _, receipt := range receipts { - receiptId, err := receiptRepository.CreateReceipt(blockId, receipt, tx) + receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, receipt, tx) if err != nil { tx.Rollback() return err @@ -68,8 +68,7 @@ func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error { return nil } -//TODO: test that creating the address should be in the transaction -func (ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { +func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { var receiptId int64 addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) if getAddressErr != nil { @@ -90,7 +89,7 @@ func (ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt, tx * return receiptId, nil } -func (receiptRepository ReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) { +func (receiptRepository FullSyncReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) { row := receiptRepository.DB.QueryRow( `SELECT contract_address_id, tx_hash, diff --git a/pkg/datastore/postgres/repositories/receipt_repository_test.go b/pkg/datastore/postgres/repositories/full_sync_receipt_repository_test.go similarity index 90% rename from pkg/datastore/postgres/repositories/receipt_repository_test.go rename to pkg/datastore/postgres/repositories/full_sync_receipt_repository_test.go index 76d1516b..0232cc99 100644 --- a/pkg/datastore/postgres/repositories/receipt_repository_test.go +++ b/pkg/datastore/postgres/repositories/full_sync_receipt_repository_test.go @@ -30,7 +30,7 @@ import ( var _ = Describe("Receipt Repository", func() { var blockRepository datastore.BlockRepository var logRepository datastore.LogRepository - var receiptRepository datastore.ReceiptRepository + var receiptRepository datastore.FullSyncReceiptRepository var db *postgres.DB var node core.Node BeforeEach(func() { @@ -44,7 +44,7 @@ var _ = Describe("Receipt Repository", func() { test_config.CleanTestDB(db) blockRepository = repositories.NewBlockRepository(db) logRepository = repositories.LogRepository{DB: db} - receiptRepository = repositories.ReceiptRepository{DB: db} + receiptRepository = repositories.FullSyncReceiptRepository{DB: db} }) Describe("Saving multiple receipts", func() { @@ -84,12 +84,12 @@ var _ = Describe("Receipt Repository", func() { Expect(err).NotTo(HaveOccurred()) - persistedReceiptOne, err := receiptRepository.GetReceipt(txHashOne) + persistedReceiptOne, err := receiptRepository.GetFullSyncReceipt(txHashOne) Expect(err).NotTo(HaveOccurred()) Expect(persistedReceiptOne).NotTo(BeNil()) Expect(persistedReceiptOne.TxHash).To(Equal(txHashOne)) - persistedReceiptTwo, err := receiptRepository.GetReceipt(txHashTwo) + persistedReceiptTwo, err := receiptRepository.GetFullSyncReceipt(txHashTwo) Expect(err).NotTo(HaveOccurred()) Expect(persistedReceiptTwo).NotTo(BeNil()) Expect(persistedReceiptTwo.TxHash).To(Equal(txHashTwo)) @@ -124,7 +124,7 @@ var _ = Describe("Receipt Repository", func() { _, err := blockRepository.CreateOrUpdateBlock(block) Expect(err).NotTo(HaveOccurred()) - receipt, err := receiptRepository.GetReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223") + receipt, err := receiptRepository.GetFullSyncReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223") Expect(err).ToNot(HaveOccurred()) //Not currently serializing bloom logs Expect(receipt.Bloom).To(Equal(core.Receipt{}.Bloom)) @@ -136,7 +136,7 @@ var _ = Describe("Receipt Repository", func() { }) It("returns ErrReceiptDoesNotExist when receipt does not exist", func() { - receipt, err := receiptRepository.GetReceipt("DOES NOT EXIST") + receipt, err := receiptRepository.GetFullSyncReceipt("DOES NOT EXIST") Expect(err).To(HaveOccurred()) Expect(receipt).To(BeZero()) }) @@ -154,7 +154,7 @@ var _ = Describe("Receipt Repository", func() { _, err := blockRepository.CreateOrUpdateBlock(block) Expect(err).NotTo(HaveOccurred()) - _, err = receiptRepository.GetReceipt(receipt.TxHash) + _, err = receiptRepository.GetFullSyncReceipt(receipt.TxHash) Expect(err).To(Not(HaveOccurred())) }) }) diff --git a/pkg/datastore/postgres/repositories/logs_repository_test.go b/pkg/datastore/postgres/repositories/logs_repository_test.go index f2f4c27b..c6544aa4 100644 --- a/pkg/datastore/postgres/repositories/logs_repository_test.go +++ b/pkg/datastore/postgres/repositories/logs_repository_test.go @@ -34,7 +34,7 @@ var _ = Describe("Logs Repository", func() { var db *postgres.DB var blockRepository datastore.BlockRepository var logsRepository datastore.LogRepository - var receiptRepository datastore.ReceiptRepository + var receiptRepository datastore.FullSyncReceiptRepository var node core.Node BeforeEach(func() { @@ -48,7 +48,7 @@ var _ = Describe("Logs Repository", func() { test_config.CleanTestDB(db) blockRepository = repositories.NewBlockRepository(db) logsRepository = repositories.LogRepository{DB: db} - receiptRepository = repositories.ReceiptRepository{DB: db} + receiptRepository = repositories.FullSyncReceiptRepository{DB: db} }) It("returns the log when it exists", func() { @@ -56,7 +56,7 @@ var _ = Describe("Logs Repository", func() { blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) Expect(err).NotTo(HaveOccurred()) tx, _ := db.Beginx() - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx) tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logsRepository.CreateLogs([]core.Log{{ @@ -94,7 +94,7 @@ var _ = Describe("Logs Repository", func() { blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) Expect(err).NotTo(HaveOccurred()) tx, _ := db.Beginx() - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx) tx.Commit() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/datastore/postgres/repositories/watched_events_repository_test.go b/pkg/datastore/postgres/repositories/watched_events_repository_test.go index f30a85f5..6e170e89 100644 --- a/pkg/datastore/postgres/repositories/watched_events_repository_test.go +++ b/pkg/datastore/postgres/repositories/watched_events_repository_test.go @@ -32,7 +32,7 @@ var _ = Describe("Watched Events Repository", func() { var blocksRepository datastore.BlockRepository var filterRepository datastore.FilterRepository var logRepository datastore.LogRepository - var receiptRepository datastore.ReceiptRepository + var receiptRepository datastore.FullSyncReceiptRepository var watchedEventRepository datastore.WatchedEventRepository BeforeEach(func() { @@ -41,7 +41,7 @@ var _ = Describe("Watched Events Repository", func() { blocksRepository = repositories.NewBlockRepository(db) filterRepository = repositories.FilterRepository{DB: db} logRepository = repositories.LogRepository{DB: db} - receiptRepository = repositories.ReceiptRepository{DB: db} + receiptRepository = repositories.FullSyncReceiptRepository{DB: db} watchedEventRepository = repositories.WatchedEventRepository{DB: db} }) @@ -80,7 +80,7 @@ var _ = Describe("Watched Events Repository", func() { blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{}) Expect(err).NotTo(HaveOccurred()) tx, _ := db.Beginx() - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx) tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logRepository.CreateLogs(logs, receiptId) @@ -139,7 +139,7 @@ var _ = Describe("Watched Events Repository", func() { blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{Hash: "Ox123"}) Expect(err).NotTo(HaveOccurred()) tx, _ := db.Beginx() - receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{}, tx) + receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx) tx.Commit() Expect(err).NotTo(HaveOccurred()) err = logRepository.CreateLogs(logs, receiptId) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index c1883ecc..5e94f7d9 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -56,10 +56,10 @@ type LogRepository interface { GetLogs(address string, blockNumber int64) ([]core.Log, error) } -type ReceiptRepository interface { +type FullSyncReceiptRepository interface { CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error - CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) //TODO: change the name to CreateReceiptInTransaction - GetReceipt(txHash string) (core.Receipt, error) + CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) + GetFullSyncReceipt(txHash string) (core.Receipt, error) } type WatchedEventRepository interface { diff --git a/pkg/fakes/mock_receipt_repository.go b/pkg/fakes/mock_receipt_repository.go index 3aad1a57..80236393 100644 --- a/pkg/fakes/mock_receipt_repository.go +++ b/pkg/fakes/mock_receipt_repository.go @@ -50,11 +50,11 @@ func (mrr *MockReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts return mrr.createReceiptsAndLogsReturnErr } -func (mrr *MockReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { +func (mrr *MockReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { panic("implement me") } -func (mrr *MockReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) { +func (mrr *MockReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) { panic("implement me") } diff --git a/pkg/geth/cold_import/importer.go b/pkg/geth/cold_import/importer.go index e29bfd22..280ed15a 100644 --- a/pkg/geth/cold_import/importer.go +++ b/pkg/geth/cold_import/importer.go @@ -26,10 +26,10 @@ type ColdImporter struct { blockRepository datastore.BlockRepository converter common.BlockConverter ethDB ethereum.Database - receiptRepository datastore.ReceiptRepository + receiptRepository datastore.FullSyncReceiptRepository } -func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.ReceiptRepository, converter common.BlockConverter) *ColdImporter { +func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.FullSyncReceiptRepository, converter common.BlockConverter) *ColdImporter { return &ColdImporter{ blockRepository: blockRepository, converter: converter, From 38f6847ff283c99a331745f2a46f1d31b3d898f1 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 5 Aug 2019 13:13:53 -0500 Subject: [PATCH 10/13] Create a HeaderSyncReceiptRepository --- .../repositories/header_repository.go | 21 --- .../repositories/header_repository_test.go | 3 +- .../header_sync_receipt_repository.go | 44 ++++++ .../header_sync_receipt_repository_test.go | 129 ++++++++++++++++++ pkg/datastore/repository.go | 4 + 5 files changed, 179 insertions(+), 22 deletions(-) create mode 100644 pkg/datastore/postgres/repositories/header_sync_receipt_repository.go create mode 100644 pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 4429fe8d..dc0baa0f 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -85,27 +85,6 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i return txId, err } -func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) { - var receiptId int64 - addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) - if getAddressErr != nil { - log.Error("createReceipt: Error getting address id: ", getAddressErr) - return receiptId, getAddressErr - } - err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts - (header_id, transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (header_id, transaction_id) DO UPDATE - SET (contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9) - RETURNING id`, - headerID, transactionID, addressId, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId) - if err != nil { - log.Error("header_repository: error inserting receipt: ", err) - return receiptId, err - } - return receiptId, err -} - func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { var header core.Header err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index bf9ea80c..dba6da09 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -216,7 +216,8 @@ var _ = Describe("Block header repository", func() { Rlp: []byte{1, 2, 3}, } - _, receiptErr := repo.CreateReceiptInTx(tx, headerID, txId, receipt) + receiptRepo := repositories.HeaderSyncReceiptRepository{} + _, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx) Expect(receiptErr).ToNot(HaveOccurred()) commitErr := tx.Commit() Expect(commitErr).ToNot(HaveOccurred()) diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go new file mode 100644 index 00000000..d23bf10d --- /dev/null +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go @@ -0,0 +1,44 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repositories + +import ( + "github.com/ethereum/go-ethereum/log" + "github.com/jmoiron/sqlx" + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type HeaderSyncReceiptRepository struct{} + +func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) { + var receiptId int64 + addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress) + if getAddressErr != nil { + log.Error("createReceipt: Error getting address id: ", getAddressErr) + return receiptId, getAddressErr + } + err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts + (header_id, transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, transaction_id) DO UPDATE + SET (contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9) + RETURNING id`, + headerID, transactionID, addressId, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId) + if err != nil { + log.Error("header_repository: error inserting receipt: ", err) + return receiptId, err + } + return receiptId, err +} diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go new file mode 100644 index 00000000..4a06554d --- /dev/null +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go @@ -0,0 +1,129 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repositories_test + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/test_config" + "math/big" +) + +var _ = Describe("Header Sync Receipt Repo", func() { + var ( + rawHeader []byte + err error + timestamp string + db *postgres.DB + receiptRepo repositories.HeaderSyncReceiptRepository + headerRepo repositories.HeaderRepository + header core.Header + ) + + BeforeEach(func() { + rawHeader, err = json.Marshal(types.Header{}) + Expect(err).NotTo(HaveOccurred()) + timestamp = big.NewInt(123456789).String() + + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + receiptRepo = repositories.HeaderSyncReceiptRepository{} + headerRepo = repositories.NewHeaderRepository(db) + header = core.Header{ + BlockNumber: 100, + Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(), + Raw: rawHeader, + Timestamp: timestamp, + } + }) + Describe("creating a receipt", func() { + It("adds a receipt in a tx", func() { + headerID, err := headerRepo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + fromAddress := common.HexToAddress("0x1234") + toAddress := common.HexToAddress("0x5678") + txHash := common.HexToHash("0x9876") + txIndex := big.NewInt(123) + transaction := core.TransactionModel{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", + } + tx, err := db.Beginx() + Expect(err).ToNot(HaveOccurred()) + txId, txErr := headerRepo.CreateTransactionInTx(tx, headerID, transaction) + Expect(txErr).ToNot(HaveOccurred()) + + contractAddr := common.HexToAddress("0x1234") + stateRoot := common.HexToHash("0x5678") + receipt := core.Receipt{ + ContractAddress: contractAddr.Hex(), + TxHash: txHash.Hex(), + GasUsed: 10, + CumulativeGasUsed: 100, + StateRoot: stateRoot.Hex(), + Rlp: []byte{1, 2, 3}, + } + + _, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx) + Expect(receiptErr).ToNot(HaveOccurred()) + commitErr := tx.Commit() + Expect(commitErr).ToNot(HaveOccurred()) + + type idModel struct { + TransactionId int64 `db:"transaction_id"` + ContractAddressId int64 `db:"contract_address_id"` + CumulativeGasUsed uint64 `db:"cumulative_gas_used"` + GasUsed uint64 `db:"gas_used"` + StateRoot string `db:"state_root"` + Status int + TxHash string `db:"tx_hash"` + Rlp []byte `db:"rlp"` + } + + var addressId int64 + getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex()) + Expect(getAddressErr).NotTo(HaveOccurred()) + + var dbReceipt idModel + getReceiptErr := db.Get(&dbReceipt, + `SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp + FROM public.header_sync_receipts WHERE header_id = $1`, headerID) + Expect(getReceiptErr).NotTo(HaveOccurred()) + + Expect(dbReceipt.TransactionId).To(Equal(txId)) + Expect(dbReceipt.TxHash).To(Equal(txHash.Hex())) + Expect(dbReceipt.ContractAddressId).To(Equal(addressId)) + Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100))) + Expect(dbReceipt.GasUsed).To(Equal(uint64(10))) + Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex())) + Expect(dbReceipt.Status).To(Equal(0)) + Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3})) + }) + }) +}) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index 5e94f7d9..fe8bad9c 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -62,6 +62,10 @@ type FullSyncReceiptRepository interface { GetFullSyncReceipt(txHash string) (core.Receipt, error) } +type HeaderSyncReceiptRepository interface { + CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) +} + type WatchedEventRepository interface { GetWatchedEvents(name string) ([]*core.WatchedEvent, error) } From 58c23c6632b11fc06e9f948922ed7e9d9a7a3ec1 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 5 Aug 2019 13:28:06 -0500 Subject: [PATCH 11/13] Update Copyrights --- cmd/compose.go | 11 ++++---- cmd/composeAndExecute.go | 12 ++++---- cmd/execute.go | 11 ++++---- .../contract_watcher_full_transformer_test.go | 16 +++++++++++ ...ct_watcher_header_sync_transformer_test.go | 16 +++++++++++ .../repositories/address_repository.go | 28 ++++++++++--------- .../repositories/address_repository_test.go | 28 ++++++++++--------- .../header_sync_receipt_repository.go | 28 ++++++++++--------- .../header_sync_receipt_repository_test.go | 28 ++++++++++--------- .../rpc/transaction_converter_test.go | 16 +++++++++++ 10 files changed, 127 insertions(+), 67 deletions(-) diff --git a/cmd/compose.go b/cmd/compose.go index 77e0e7ad..f92db41a 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -1,17 +1,18 @@ -// Copyright © 2019 Vulcanize, Inc -// +// VulcanizeDB +// Copyright © 2019 Vulcanize + // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// + // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. -// + // You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// along with this program. If not, see . package cmd diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 680bb0bb..4dc11385 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -1,17 +1,19 @@ -// Copyright © 2019 Vulcanize, Inc -// +// VulcanizeDB +// Copyright © 2019 Vulcanize + // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// + // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. -// + // You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// along with this program. If not, see . + package cmd diff --git a/cmd/execute.go b/cmd/execute.go index 337d306c..c6d7bb15 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -1,17 +1,18 @@ -// Copyright © 2019 Vulcanize, Inc -// +// VulcanizeDB +// Copyright © 2019 Vulcanize + // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// + // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. -// + // You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +// along with this program. If not, see . package cmd diff --git a/integration_test/contract_watcher_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go index e972d067..4e0cce01 100644 --- a/integration_test/contract_watcher_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -1,3 +1,19 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package integration import ( diff --git a/integration_test/contract_watcher_header_sync_transformer_test.go b/integration_test/contract_watcher_header_sync_transformer_test.go index 1c8507f2..3b127e84 100644 --- a/integration_test/contract_watcher_header_sync_transformer_test.go +++ b/integration_test/contract_watcher_header_sync_transformer_test.go @@ -1,3 +1,19 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package integration import ( diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index da25bda6..86001764 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -1,16 +1,18 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . package repositories diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index 1ac53672..baa2fcb7 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -1,16 +1,18 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . package repositories_test diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go index d23bf10d..16b87644 100644 --- a/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository.go @@ -1,16 +1,18 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . package repositories diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go index 4a06554d..2f6bc910 100644 --- a/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go @@ -1,16 +1,18 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . package repositories_test diff --git a/pkg/geth/converters/rpc/transaction_converter_test.go b/pkg/geth/converters/rpc/transaction_converter_test.go index 21047c92..bcd7b725 100644 --- a/pkg/geth/converters/rpc/transaction_converter_test.go +++ b/pkg/geth/converters/rpc/transaction_converter_test.go @@ -1,3 +1,19 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package rpc_test import ( From 1b3786338f649fe7d0faa7f77b9d76cf94f4b9f5 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 19 Aug 2019 16:51:17 -0500 Subject: [PATCH 12/13] Address PR feedback --- cmd/composeAndExecute.go | 1 - .../postgres/repositories/address_repository.go | 7 +++---- .../repositories/address_repository_test.go | 4 +++- .../header_sync_receipt_repository_test.go | 16 +++++++++------- .../watched_events_repository_test.go | 4 +++- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index 4dc11385..d3e0e0b9 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - package cmd import ( diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 86001764..3976fa28 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -18,8 +18,10 @@ package repositories import ( "database/sql" + "github.com/ethereum/go-ethereum/common" "github.com/jmoiron/sqlx" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -56,8 +58,5 @@ func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address st func (AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error) { var address string getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) - if getErr != nil { - return "", getErr - } - return address, nil + return address, getErr } diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index baa2fcb7..5b67cfd0 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -17,14 +17,16 @@ package repositories_test import ( + "strings" + "github.com/jmoiron/sqlx" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/test_config" - "strings" ) var _ = Describe("address lookup", func() { diff --git a/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go index 2f6bc910..f88abdc0 100644 --- a/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_sync_receipt_repository_test.go @@ -18,26 +18,28 @@ package repositories_test import ( "encoding/json" + "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/test_config" - "math/big" ) var _ = Describe("Header Sync Receipt Repo", func() { var ( - rawHeader []byte - err error - timestamp string - db *postgres.DB + rawHeader []byte + err error + timestamp string + db *postgres.DB receiptRepo repositories.HeaderSyncReceiptRepository - headerRepo repositories.HeaderRepository - header core.Header + headerRepo repositories.HeaderRepository + header core.Header ) BeforeEach(func() { diff --git a/pkg/datastore/postgres/repositories/watched_events_repository_test.go b/pkg/datastore/postgres/repositories/watched_events_repository_test.go index 6e170e89..23ff8af6 100644 --- a/pkg/datastore/postgres/repositories/watched_events_repository_test.go +++ b/pkg/datastore/postgres/repositories/watched_events_repository_test.go @@ -19,6 +19,7 @@ package repositories_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -79,7 +80,8 @@ var _ = Describe("Watched Events Repository", func() { Expect(err).ToNot(HaveOccurred()) blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{}) Expect(err).NotTo(HaveOccurred()) - tx, _ := db.Beginx() + tx, txBeginErr := db.Beginx() + Expect(txBeginErr).NotTo(HaveOccurred()) receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx) tx.Commit() Expect(err).NotTo(HaveOccurred()) From edc0bdf66840fb28ce4392922c2056d744e8a93a Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Mon, 19 Aug 2019 16:55:38 -0500 Subject: [PATCH 13/13] Address more PR feedback --- .../repositories/address_repository.go | 10 +++++----- .../repositories/address_repository_test.go | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/datastore/postgres/repositories/address_repository.go b/pkg/datastore/postgres/repositories/address_repository.go index 3976fa28..7cf709d2 100644 --- a/pkg/datastore/postgres/repositories/address_repository.go +++ b/pkg/datastore/postgres/repositories/address_repository.go @@ -27,11 +27,11 @@ import ( type AddressRepository struct{} -func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int, error) { +func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() - var addressId int + var addressId int64 getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) if getErr == sql.ErrNoRows { insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) @@ -41,11 +41,11 @@ func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (in return addressId, getErr } -func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int, error) { +func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) { stringAddressToCommonAddress := common.HexToAddress(address) hexAddress := stringAddressToCommonAddress.Hex() - var addressId int + var addressId int64 getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress) if getErr == sql.ErrNoRows { insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId) @@ -55,7 +55,7 @@ func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address st return addressId, getErr } -func (AddressRepository) GetAddressById(db *postgres.DB, id int) (string, error) { +func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) { var address string getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id) return address, getErr diff --git a/pkg/datastore/postgres/repositories/address_repository_test.go b/pkg/datastore/postgres/repositories/address_repository_test.go index 5b67cfd0..1b2b788f 100644 --- a/pkg/datastore/postgres/repositories/address_repository_test.go +++ b/pkg/datastore/postgres/repositories/address_repository_test.go @@ -41,8 +41,12 @@ var _ = Describe("address lookup", func() { repo = repositories.AddressRepository{} }) + AfterEach(func() { + test_config.CleanTestDB(db) + }) + type dbAddress struct { - Id int + Id int64 Address string } @@ -59,16 +63,17 @@ var _ = Describe("address lookup", func() { }) It("returns the existing record id if the address already exists", func() { - _, createErr := repo.GetOrCreateAddress(db, address) + createId, createErr := repo.GetOrCreateAddress(db, address) Expect(createErr).NotTo(HaveOccurred()) - _, getErr := repo.GetOrCreateAddress(db, address) + getId, getErr := repo.GetOrCreateAddress(db, address) Expect(getErr).NotTo(HaveOccurred()) var addressCount int addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`) Expect(addressErr).NotTo(HaveOccurred()) Expect(addressCount).To(Equal(1)) + Expect(createId).To(Equal(getId)) }) It("gets upper-cased addresses", func() { @@ -102,10 +107,15 @@ var _ = Describe("address lookup", func() { Expect(txErr).NotTo(HaveOccurred()) }) + AfterEach(func() { + tx.Rollback() + }) + It("creates an address record", func() { addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address) Expect(createErr).NotTo(HaveOccurred()) - tx.Commit() + commitErr := tx.Commit() + Expect(commitErr).NotTo(HaveOccurred()) var actualAddress dbAddress getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)