From d93006321b880df9cb50fe94aa00cc6249efa881 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 19 Mar 2019 14:11:26 -0500 Subject: [PATCH 1/5] Rename transactions to full_sync_transactions - Table has foreign key to blocks - Add transaction RLP and transaction index to table - Enables other tables with standalone transactions or transactions associated with other data structures (e.g. headers) --- ...2_create_full_sync_transactions_table.sql} | 10 +- ...ock_id_index_to_full_sync_transactions.sql | 5 + ...009_add_block_id_index_to_transactions.sql | 5 - ..._tx_to_index_to_full_sync_transactions.sql | 5 + .../00011_add_tx_to_index_to_transactions.sql | 5 - ...x_from_index_to_full_sync_transactions.sql | 5 + ...0012_add_tx_from_index_to_transactions.sql | 5 - db/migrations/00013_create_receipts_table.sql | 7 +- .../00020_associate_receipts_with_blocks.sql | 6 +- db/schema.sql | 130 ++++++------- .../contract_watcher_full_transformer_test.go | 24 ++- ...contract_watcher_light_transformer_test.go | 12 +- .../full/retriever/block_retriever_test.go | 76 ++++++-- .../shared/helpers/test_helpers/database.go | 2 +- .../helpers/test_helpers/mocks/entities.go | 28 ++- pkg/core/transaction.go | 14 +- pkg/datastore/postgres/errors.go | 38 ++++ pkg/datastore/postgres/postgres.go | 23 +-- pkg/datastore/postgres/postgres_test.go | 8 +- .../postgres/repositories/block_repository.go | 79 ++++---- .../repositories/block_repository_test.go | 172 +++++++++++------- .../repositories/contract_repository.go | 4 +- .../postgres/repositories/logs_repository.go | 14 +- .../repositories/logs_repository_test.go | 7 +- .../repositories/receipts_repository_test.go | 11 +- pkg/fakes/data.go | 40 ++++ .../converters/common/block_converter_test.go | 6 + .../converters/rpc/transaction_converter.go | 25 ++- pkg/plugin/test_helpers/database.go | 2 +- test_config/test_config.go | 2 +- 30 files changed, 498 insertions(+), 272 deletions(-) rename db/migrations/{00002_create_transactions_table.sql => 00002_create_full_sync_transactions_table.sql} (74%) create mode 100644 db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql delete mode 100644 db/migrations/00009_add_block_id_index_to_transactions.sql create mode 100644 db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql delete mode 100644 db/migrations/00011_add_tx_to_index_to_transactions.sql create mode 100644 db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql delete mode 100644 db/migrations/00012_add_tx_from_index_to_transactions.sql create mode 100644 pkg/datastore/postgres/errors.go diff --git a/db/migrations/00002_create_transactions_table.sql b/db/migrations/00002_create_full_sync_transactions_table.sql similarity index 74% rename from db/migrations/00002_create_transactions_table.sql rename to db/migrations/00002_create_full_sync_transactions_table.sql index 80aa4c7c..f1e0be23 100644 --- a/db/migrations/00002_create_transactions_table.sql +++ b/db/migrations/00002_create_full_sync_transactions_table.sql @@ -1,16 +1,18 @@ -- +goose Up -CREATE TABLE transactions ( +CREATE TABLE full_sync_transactions ( id SERIAL PRIMARY KEY, block_id INTEGER NOT NULL REFERENCES blocks(id) ON DELETE CASCADE, - input_data VARCHAR, - tx_from VARCHAR(66), gaslimit NUMERIC, gasprice NUMERIC, hash VARCHAR(66), + input_data VARCHAR, nonce NUMERIC, + raw BYTEA, + tx_from VARCHAR(66), + tx_index INTEGER, tx_to VARCHAR(66), "value" NUMERIC ); -- +goose Down -DROP TABLE transactions; \ No newline at end of file +DROP TABLE full_sync_transactions; \ No newline at end of file diff --git a/db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql b/db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql new file mode 100644 index 00000000..99b2ca78 --- /dev/null +++ b/db/migrations/00009_add_block_id_index_to_full_sync_transactions.sql @@ -0,0 +1,5 @@ +-- +goose Up +CREATE INDEX block_id_index ON full_sync_transactions (block_id); + +-- +goose Down +DROP INDEX block_id_index; diff --git a/db/migrations/00009_add_block_id_index_to_transactions.sql b/db/migrations/00009_add_block_id_index_to_transactions.sql deleted file mode 100644 index 318cc9be..00000000 --- a/db/migrations/00009_add_block_id_index_to_transactions.sql +++ /dev/null @@ -1,5 +0,0 @@ --- +goose Up -CREATE INDEX block_id_index ON transactions (block_id); - --- +goose Down -DROP INDEX block_id_index; diff --git a/db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql b/db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql new file mode 100644 index 00000000..cf2977d1 --- /dev/null +++ b/db/migrations/00011_add_tx_to_index_to_full_sync_transactions.sql @@ -0,0 +1,5 @@ +-- +goose Up +CREATE INDEX tx_to_index ON full_sync_transactions(tx_to); + +-- +goose Down +DROP INDEX tx_to_index; diff --git a/db/migrations/00011_add_tx_to_index_to_transactions.sql b/db/migrations/00011_add_tx_to_index_to_transactions.sql deleted file mode 100644 index fa50de5e..00000000 --- a/db/migrations/00011_add_tx_to_index_to_transactions.sql +++ /dev/null @@ -1,5 +0,0 @@ --- +goose Up -CREATE INDEX tx_to_index ON transactions(tx_to); - --- +goose Down -DROP INDEX tx_to_index; diff --git a/db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql b/db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql new file mode 100644 index 00000000..fa6f0543 --- /dev/null +++ b/db/migrations/00012_add_tx_from_index_to_full_sync_transactions.sql @@ -0,0 +1,5 @@ +-- +goose Up +CREATE INDEX tx_from_index ON full_sync_transactions(tx_from); + +-- +goose Down +DROP INDEX tx_from_index; diff --git a/db/migrations/00012_add_tx_from_index_to_transactions.sql b/db/migrations/00012_add_tx_from_index_to_transactions.sql deleted file mode 100644 index 6d052bae..00000000 --- a/db/migrations/00012_add_tx_from_index_to_transactions.sql +++ /dev/null @@ -1,5 +0,0 @@ --- +goose Up -CREATE INDEX tx_from_index ON transactions(tx_from); - --- +goose Down -DROP INDEX tx_from_index; diff --git a/db/migrations/00013_create_receipts_table.sql b/db/migrations/00013_create_receipts_table.sql index 53ef4276..af3424ac 100644 --- a/db/migrations/00013_create_receipts_table.sql +++ b/db/migrations/00013_create_receipts_table.sql @@ -2,16 +2,13 @@ CREATE TABLE receipts ( id SERIAL PRIMARY KEY, - transaction_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE, contract_address VARCHAR(42), cumulative_gas_used NUMERIC, gas_used NUMERIC, state_root VARCHAR(66), status INTEGER, - tx_hash VARCHAR(66), - CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) - REFERENCES transactions (id) - ON DELETE CASCADE + tx_hash VARCHAR(66) ); diff --git a/db/migrations/00020_associate_receipts_with_blocks.sql b/db/migrations/00020_associate_receipts_with_blocks.sql index 889772ff..3b31e2c0 100644 --- a/db/migrations/00020_associate_receipts_with_blocks.sql +++ b/db/migrations/00020_associate_receipts_with_blocks.sql @@ -4,7 +4,7 @@ ALTER TABLE receipts UPDATE receipts SET block_id = ( - SELECT block_id FROM transactions WHERE transactions.id = receipts.transaction_id + SELECT block_id FROM full_sync_transactions WHERE full_sync_transactions.id = receipts.transaction_id ); ALTER TABLE receipts @@ -28,7 +28,7 @@ CREATE INDEX transaction_id_index ON receipts (transaction_id); UPDATE receipts SET transaction_id = ( - SELECT id FROM transactions WHERE transactions.hash = receipts.tx_hash + SELECT id FROM full_sync_transactions WHERE full_sync_transactions.hash = receipts.tx_hash ); ALTER TABLE receipts @@ -37,7 +37,7 @@ ALTER TABLE receipts ALTER TABLE receipts ADD CONSTRAINT transaction_fk FOREIGN KEY (transaction_id) -REFERENCES transactions (id) +REFERENCES full_sync_transactions (id) ON DELETE CASCADE; ALTER TABLE receipts diff --git a/db/schema.sql b/db/schema.sql index 6e45cbaa..d6a05de6 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -151,6 +151,46 @@ CREATE TABLE public.eth_nodes ( ); +-- +-- Name: full_sync_transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.full_sync_transactions ( + id integer NOT NULL, + block_id integer NOT NULL, + gaslimit numeric, + gasprice numeric, + hash character varying(66), + input_data character varying, + nonce numeric, + raw bytea, + tx_from character varying(66), + tx_index integer, + tx_to character varying(66), + value numeric +); + + +-- +-- Name: full_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.full_sync_transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: full_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.full_sync_transactions_id_seq OWNED BY public.full_sync_transactions.id; + + -- -- Name: goose_db_version; Type: TABLE; Schema: public; Owner: - -- @@ -368,44 +408,6 @@ CREATE SEQUENCE public.receipts_id_seq ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id; --- --- Name: transactions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.transactions ( - id integer NOT NULL, - block_id integer NOT NULL, - input_data character varying, - tx_from character varying(66), - gaslimit numeric, - gasprice numeric, - hash character varying(66), - nonce numeric, - tx_to character varying(66), - value numeric -); - - --- --- Name: transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.transactions_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.transactions_id_seq OWNED BY public.transactions.id; - - -- -- Name: watched_contracts; Type: TABLE; Schema: public; Owner: - -- @@ -481,6 +483,13 @@ ALTER TABLE ONLY public.checked_headers ALTER COLUMN id SET DEFAULT nextval('pub ALTER TABLE ONLY public.eth_nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass); +-- +-- Name: full_sync_transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.full_sync_transactions_id_seq'::regclass); + + -- -- Name: goose_db_version id; Type: DEFAULT; Schema: public; Owner: - -- @@ -523,13 +532,6 @@ ALTER TABLE ONLY public.queued_storage ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.receipts ALTER COLUMN id SET DEFAULT nextval('public.receipts_id_seq'::regclass); --- --- Name: transactions id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions ALTER COLUMN id SET DEFAULT nextval('public.transactions_id_seq'::regclass); - - -- -- Name: watched_contracts contract_id; Type: DEFAULT; Schema: public; Owner: - -- @@ -577,6 +579,14 @@ ALTER TABLE ONLY public.eth_nodes ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id); +-- +-- Name: full_sync_transactions full_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.full_sync_transactions + ADD CONSTRAINT full_sync_transactions_pkey PRIMARY KEY (id); + + -- -- Name: goose_db_version goose_db_version_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -633,14 +643,6 @@ ALTER TABLE ONLY public.receipts ADD CONSTRAINT receipts_pkey PRIMARY KEY (id); --- --- Name: transactions transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_pkey PRIMARY KEY (id); - - -- -- Name: watched_contracts watched_contracts_contract_hash_key; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -661,7 +663,7 @@ ALTER TABLE ONLY public.watched_contracts -- Name: block_id_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX block_id_index ON public.transactions USING btree (block_id); +CREATE INDEX block_id_index ON public.full_sync_transactions USING btree (block_id); -- @@ -689,14 +691,14 @@ CREATE INDEX number_index ON public.blocks USING btree (number); -- Name: tx_from_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX tx_from_index ON public.transactions USING btree (tx_from); +CREATE INDEX tx_from_index ON public.full_sync_transactions USING btree (tx_from); -- -- Name: tx_to_index; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX tx_to_index ON public.transactions USING btree (tx_to); +CREATE INDEX tx_to_index ON public.full_sync_transactions USING btree (tx_to); -- @@ -723,6 +725,14 @@ ALTER TABLE ONLY public.headers ADD CONSTRAINT eth_nodes_fk FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE; +-- +-- Name: full_sync_transactions full_sync_transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.full_sync_transactions + ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE; + + -- -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -739,14 +749,6 @@ ALTER TABLE ONLY public.logs ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES public.receipts(id) ON DELETE CASCADE; --- --- Name: transactions transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE; - - -- -- PostgreSQL database dump complete -- diff --git a/integration_test/contract_watcher_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go index bcb7a29a..943428f2 100644 --- a/integration_test/contract_watcher_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -40,8 +40,10 @@ var _ = Describe("contractWatcher full transformer", func() { Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + _, insertErr := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + Expect(insertErrTwo).NotTo(HaveOccurred()) t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -63,8 +65,10 @@ var _ = Describe("contractWatcher full transformer", func() { }) It("Does nothing if watched events are unset", func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + _, insertErr := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + Expect(insertErrTwo).NotTo(HaveOccurred()) var testConf config.ContractConfig testConf = test_helpers.TusdConfig testConf.Events = nil @@ -80,8 +84,10 @@ var _ = Describe("contractWatcher full transformer", func() { Describe("Execute", func() { BeforeEach(func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + _, insertErr := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + Expect(insertErrTwo).NotTo(HaveOccurred()) }) It("Transforms watched contract data into custom repositories", func() { @@ -181,8 +187,10 @@ var _ = Describe("contractWatcher full transformer", func() { Describe("Execute- against ENS registry contract", func() { BeforeEach(func() { - blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1) - blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2) + _, insertErr := blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2) + Expect(insertErrTwo).NotTo(HaveOccurred()) }) It("Transforms watched contract data into custom repositories", func() { diff --git a/integration_test/contract_watcher_light_transformer_test.go b/integration_test/contract_watcher_light_transformer_test.go index cf9efa77..3aaa3267 100644 --- a/integration_test/contract_watcher_light_transformer_test.go +++ b/integration_test/contract_watcher_light_transformer_test.go @@ -38,8 +38,10 @@ var _ = Describe("contractWatcher light transformer", func() { Describe("Init", func() { It("Initializes transformer's contract objects", func() { - headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) - headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + _, insertErr := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + Expect(insertErrTwo).NotTo(HaveOccurred()) t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -61,8 +63,10 @@ var _ = Describe("contractWatcher light transformer", func() { }) It("Does nothing if nothing if no addresses are configured", func() { - headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) - headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + _, insertErr := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + Expect(insertErr).NotTo(HaveOccurred()) + _, insertErrTwo := headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + Expect(insertErrTwo).NotTo(HaveOccurred()) var testConf config.ContractConfig testConf = test_helpers.TusdConfig testConf.Addresses = nil diff --git a/pkg/contract_watcher/full/retriever/block_retriever_test.go b/pkg/contract_watcher/full/retriever/block_retriever_test.go index 4cc4c04e..4b331983 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/full/retriever/block_retriever_test.go @@ -17,6 +17,7 @@ package retriever_test import ( + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "strings" @@ -32,6 +33,7 @@ import ( var _ = Describe("Block Retriever", func() { var db *postgres.DB var r retriever.BlockRetriever + var rawTransaction []byte var blockRepository repositories.BlockRepository // Contains no contract address @@ -45,6 +47,10 @@ var _ = Describe("Block Retriever", func() { db, _ = test_helpers.SetupDBandBC() blockRepository = *repositories.NewBlockRepository(db) r = retriever.NewBlockRetriever(db) + gethTransaction := types.Transaction{} + var err error + rawTransaction, err = gethTransaction.MarshalJSON() + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -53,18 +59,23 @@ var _ = Describe("Block Retriever", func() { Describe("RetrieveFirstBlock", func() { It("Retrieves block number where contract first appears in receipt, if available", func() { - // Contains the address in the receipt block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + GasPrice: 0, + GasLimit: 0, + Nonce: 0, + Raw: rawTransaction, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", ContractAddress: constants.TusdContractAddress, Logs: []core.Log{}, }, + TxIndex: 0, + Value: "0", }}, } @@ -73,7 +84,11 @@ var _ = Describe("Block Retriever", func() { Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + GasPrice: 0, + GasLimit: 0, + Nonce: 0, + Raw: rawTransaction, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", ContractAddress: constants.TusdContractAddress, @@ -91,12 +106,17 @@ var _ = Describe("Block Retriever", func() { Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", }}, }, + TxIndex: 0, + Value: "0", }}, } - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) - blockRepository.CreateOrUpdateBlock(block3) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := blockRepository.CreateOrUpdateBlock(block3) + Expect(insertErrThree).NotTo(HaveOccurred()) i, err := r.RetrieveFirstBlock(strings.ToLower(constants.TusdContractAddress)) Expect(err).NotTo(HaveOccurred()) @@ -104,12 +124,15 @@ var _ = Describe("Block Retriever", func() { }) It("Retrieves block number where contract first appears in event logs if it cannot find the address in a receipt", func() { - block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + GasPrice: 0, + GasLimit: 0, + Nonce: 0, + Raw: rawTransaction, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", ContractAddress: "", @@ -127,6 +150,8 @@ var _ = Describe("Block Retriever", func() { Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", }}, }, + TxIndex: 0, + Value: "0", }}, } @@ -134,7 +159,11 @@ var _ = Describe("Block Retriever", func() { Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + GasPrice: 0, + GasLimit: 0, + Nonce: 0, + Raw: rawTransaction, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", ContractAddress: "", @@ -152,12 +181,17 @@ var _ = Describe("Block Retriever", func() { Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", }}, }, + TxIndex: 0, + Value: "0", }}, } - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) - blockRepository.CreateOrUpdateBlock(block3) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := blockRepository.CreateOrUpdateBlock(block3) + Expect(insertErrThree).NotTo(HaveOccurred()) i, err := r.RetrieveFirstBlock(constants.DaiContractAddress) Expect(err).NotTo(HaveOccurred()) @@ -177,9 +211,12 @@ var _ = Describe("Block Retriever", func() { Transactions: []core.Transaction{}, } - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) - blockRepository.CreateOrUpdateBlock(block3) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := blockRepository.CreateOrUpdateBlock(block3) + Expect(insertErrThree).NotTo(HaveOccurred()) _, err := r.RetrieveFirstBlock(constants.DaiContractAddress) Expect(err).To(HaveOccurred()) @@ -200,9 +237,12 @@ var _ = Describe("Block Retriever", func() { Transactions: []core.Transaction{}, } - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) - blockRepository.CreateOrUpdateBlock(block3) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(block2) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := blockRepository.CreateOrUpdateBlock(block3) + Expect(insertErrThree).NotTo(HaveOccurred()) i, err := r.RetrieveMostRecentBlock() Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index 93c79459..c33a304e 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -237,7 +237,7 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM log_filters`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM transactions`) + _, err = tx.Exec(`DELETE FROM full_sync_transactions`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM receipts`) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index f9224847..8bb37ce8 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -34,7 +34,10 @@ var TransferBlock1 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 6194633, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", + GasLimit: 0, + GasPrice: 0, + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", + Nonce: 0, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", ContractAddress: "", @@ -52,6 +55,8 @@ var TransferBlock1 = core.Block{ Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", }}, }, + TxIndex: 0, + Value: "0", }}, } @@ -59,7 +64,10 @@ var TransferBlock2 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ooo", Number: 6194634, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", + GasLimit: 0, + GasPrice: 0, + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", + Nonce: 0, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", ContractAddress: "", @@ -77,6 +85,8 @@ var TransferBlock2 = core.Block{ Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", }}, }, + TxIndex: 0, + Value: "0", }}, } @@ -84,7 +94,10 @@ var NewOwnerBlock1 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ppp", Number: 6194635, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb", + GasLimit: 0, + GasPrice: 0, + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb", + Nonce: 0, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb", ContractAddress: "", @@ -102,6 +115,8 @@ var NewOwnerBlock1 = core.Block{ Data: "0x000000000000000000000000000000000000000000000000000000000000af21", }}, }, + TxIndex: 0, + Value: "0", }}, } @@ -109,7 +124,10 @@ var NewOwnerBlock2 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ggg", Number: 6194636, Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll", + GasLimit: 0, + GasPrice: 0, + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll", + Nonce: 0, Receipt: core.Receipt{ TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll", ContractAddress: "", @@ -127,6 +145,8 @@ var NewOwnerBlock2 = core.Block{ Data: "0x000000000000000000000000000000000000000000000000000000000000af21", }}, }, + TxIndex: 0, + Value: "0", }}, } diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index dc5f21b6..6094557c 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -17,13 +17,15 @@ package core type Transaction struct { - Hash string `db:"hash"` Data string `db:"input_data"` - Nonce uint64 `db:"nonce"` - To string `db:"tx_to"` From string `db:"tx_from"` - GasLimit uint64 `db:"gaslimit"` - GasPrice int64 `db:"gasprice"` + GasLimit uint64 + GasPrice int64 + Hash string + Nonce uint64 + Raw []byte Receipt - Value string `db:"value"` + To string `db:"tx_to"` + TxIndex int64 `db:"tx_index"` + Value string } diff --git a/pkg/datastore/postgres/errors.go b/pkg/datastore/postgres/errors.go new file mode 100644 index 00000000..ffb23e78 --- /dev/null +++ b/pkg/datastore/postgres/errors.go @@ -0,0 +1,38 @@ +package postgres + +import ( + "errors" + "fmt" +) + +const ( + BeginTransactionFailedMsg = "failed to begin transaction" + DbConnectionFailedMsg = "db connection failed" + DeleteQueryFailedMsg = "delete query failed" + InsertQueryFailedMsg = "insert query failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrBeginTransactionFailed(beginErr error) error { + return formatError(BeginTransactionFailedMsg, beginErr.Error()) +} + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrDBDeleteFailed(deleteErr error) error { + return formatError(DeleteQueryFailedMsg, deleteErr.Error()) +} + +func ErrDBInsertFailed(insertErr error) error { + return formatError(InsertQueryFailedMsg, insertErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return errors.New(fmt.Sprintf("%s: %s", msg, err)) +} diff --git a/pkg/datastore/postgres/postgres.go b/pkg/datastore/postgres/postgres.go index d6472652..66cf535f 100644 --- a/pkg/datastore/postgres/postgres.go +++ b/pkg/datastore/postgres/postgres.go @@ -17,8 +17,6 @@ package postgres import ( - "errors" - "github.com/jmoiron/sqlx" _ "github.com/lib/pq" //postgres driver "github.com/vulcanize/vulcanizedb/pkg/config" @@ -31,23 +29,18 @@ type DB struct { NodeID int64 } -var ( - ErrDBInsertFailed = errors.New("postgres: insert failed") - ErrDBDeleteFailed = errors.New("postgres: delete failed") - ErrDBConnectionFailed = errors.New("postgres: db connection failed") - ErrUnableToSetNode = errors.New("postgres: unable to set node") -) +var () func NewDB(databaseConfig config.Database, node core.Node) (*DB, error) { connectString := config.DbConnectionString(databaseConfig) - db, err := sqlx.Connect("postgres", connectString) - if err != nil { - return &DB{}, ErrDBConnectionFailed + db, connectErr := sqlx.Connect("postgres", connectString) + if connectErr != nil { + return &DB{}, ErrDBConnectionFailed(connectErr) } pg := DB{DB: db, Node: node} - err = pg.CreateNode(&node) - if err != nil { - return &DB{}, ErrUnableToSetNode + nodeErr := pg.CreateNode(&node) + if nodeErr != nil { + return &DB{}, ErrUnableToSetNode(nodeErr) } return &pg, nil } @@ -66,7 +59,7 @@ func (db *DB) CreateNode(node *core.Node) error { RETURNING id`, node.GenesisBlock, node.NetworkID, node.ID, node.ClientName).Scan(&nodeId) if err != nil { - return ErrUnableToSetNode + return ErrUnableToSetNode(err) } db.NodeID = nodeId return nil diff --git a/pkg/datastore/postgres/postgres_test.go b/pkg/datastore/postgres/postgres_test.go index ba6ecb77..50dd8b6d 100644 --- a/pkg/datastore/postgres/postgres_test.go +++ b/pkg/datastore/postgres/postgres_test.go @@ -109,14 +109,18 @@ var _ = Describe("Postgres DB", func() { _, err := postgres.NewDB(invalidDatabase, node) - Expect(err).To(Equal(postgres.ErrDBConnectionFailed)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(postgres.DbConnectionFailedMsg)) }) It("throws error when can't create node", func() { badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) node := core.Node{GenesisBlock: badHash, NetworkID: 1, ID: "x123", ClientName: "geth"} + _, err := postgres.NewDB(test_config.DBConfig, node) - Expect(err).To(Equal(postgres.ErrUnableToSetNode)) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(postgres.SettingNodeFailedMsg)) }) It("does not commit log if log is invalid", func() { diff --git a/pkg/datastore/postgres/repositories/block_repository.go b/pkg/datastore/postgres/repositories/block_repository.go index 4a778aea..56d72f33 100644 --- a/pkg/datastore/postgres/repositories/block_repository.go +++ b/pkg/datastore/postgres/repositories/block_repository.go @@ -121,26 +121,42 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block, func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) { var blockId int64 - tx, _ := blockRepository.database.Beginx() - err := tx.QueryRow( + tx, beginErr := blockRepository.database.Beginx() + if beginErr != nil { + return 0, postgres.ErrBeginTransactionFailed(beginErr) + } + insertBlockErr := tx.QueryRow( `INSERT INTO blocks (eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING id `, blockRepository.database.NodeID, block.Number, block.GasLimit, block.GasUsed, block.Time, block.Difficulty, block.Hash, block.Nonce, block.ParentHash, block.Size, block.UncleHash, block.IsFinal, block.Miner, block.ExtraData, block.Reward, block.UnclesReward, blockRepository.database.Node.ID). Scan(&blockId) - if err != nil { - tx.Rollback() - return 0, err + if insertBlockErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Error("failed to rollback transaction: ", rollbackErr) + } + return 0, postgres.ErrDBInsertFailed(insertBlockErr) } if len(block.Transactions) > 0 { - err = blockRepository.createTransactions(tx, blockId, block.Transactions) - if err != nil { - tx.Rollback() - return 0, postgres.ErrDBInsertFailed + insertTxErr := blockRepository.createTransactions(tx, blockId, block.Transactions) + if insertTxErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Warn("failed to rollback transaction: ", rollbackErr) + } + return 0, postgres.ErrDBInsertFailed(insertTxErr) } } - tx.Commit() + commitErr := tx.Commit() + if commitErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + log.Warn("failed to rollback transaction: ", rollbackErr) + } + return 0, commitErr + } return blockId, nil } @@ -166,11 +182,11 @@ func nullStringToZero(s string) string { func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error { _, err := tx.Exec( - `INSERT INTO transactions - (block_id, hash, nonce, tx_to, tx_from, gaslimit, gasprice, value, input_data) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::NUMERIC, $9) - RETURNING id`, - blockId, transaction.Hash, transaction.Nonce, transaction.To, transaction.From, transaction.GasLimit, transaction.GasPrice, nullStringToZero(transaction.Value), transaction.Data) + `INSERT INTO full_sync_transactions + (block_id, gaslimit, gasprice, hash, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") + VALUES ($1, $2::NUMERIC, $3::NUMERIC, $4, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC) + RETURNING id`, blockId, transaction.GasLimit, transaction.GasPrice, transaction.Hash, transaction.Data, + transaction.Nonce, transaction.Raw, transaction.From, transaction.TxIndex, transaction.To, transaction.Value) if err != nil { return err } @@ -215,6 +231,7 @@ func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) { var retrievedBlockHash string + // TODO: handle possible error blockRepository.database.Get(&retrievedBlockHash, `SELECT hash FROM blocks @@ -232,7 +249,7 @@ func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, ) if err != nil { - return postgres.ErrDBInsertFailed + return postgres.ErrDBInsertFailed(err) } } return nil @@ -244,12 +261,10 @@ func blockExists(retrievedBlockHash string) bool { func (blockRepository BlockRepository) removeBlock(blockNumber int64) error { _, err := blockRepository.database.Exec( - `DELETE FROM - blocks - WHERE number=$1 AND eth_node_id=$2`, + `DELETE FROM blocks WHERE number=$1 AND eth_node_id=$2`, blockNumber, blockRepository.database.NodeID) if err != nil { - return postgres.ErrDBDeleteFailed + return postgres.ErrDBDeleteFailed(err) } return nil } @@ -266,17 +281,19 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc return core.Block{}, err } transactionRows, err := blockRepository.database.Queryx(` - SELECT hash, - nonce, - tx_to, - tx_from, - gaslimit, - gasprice, - value, - input_data - FROM transactions - WHERE block_id = $1 - ORDER BY hash`, block.ID) + SELECT hash, + gaslimit, + gasprice, + input_data, + nonce, + raw, + tx_from, + tx_index, + tx_to, + value + FROM full_sync_transactions + WHERE block_id = $1 + ORDER BY hash`, block.ID) if err != nil { log.Error("loadBlock: error fetting transactions: ", err) return core.Block{}, err diff --git a/pkg/datastore/postgres/repositories/block_repository_test.go b/pkg/datastore/postgres/repositories/block_repository_test.go index 4195f74e..14044209 100644 --- a/pkg/datastore/postgres/repositories/block_repository_test.go +++ b/pkg/datastore/postgres/repositories/block_repository_test.go @@ -17,6 +17,10 @@ package repositories_test import ( + "bytes" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "strconv" @@ -51,7 +55,8 @@ var _ = Describe("Saving blocks", func() { block := core.Block{ Number: 123, } - blockRepository.CreateOrUpdateBlock(block) + _, insertErr := blockRepository.CreateOrUpdateBlock(block) + Expect(insertErr).NotTo(HaveOccurred()) nodeTwo := core.Node{ GenesisBlock: "0x456", NetworkID: 1, @@ -98,8 +103,9 @@ var _ = Describe("Saving blocks", func() { UnclesReward: unclesReward, } - blockRepository.CreateOrUpdateBlock(block) + _, insertErr := blockRepository.CreateOrUpdateBlock(block) + Expect(insertErr).NotTo(HaveOccurred()) savedBlock, err := blockRepository.GetBlock(blockNumber) Expect(err).NotTo(HaveOccurred()) Expect(savedBlock.Reward).To(Equal(blockReward)) @@ -127,42 +133,54 @@ var _ = Describe("Saving blocks", func() { It("saves one transaction associated to the block", func() { block := core.Block{ Number: 123, - Transactions: []core.Transaction{{}}, + Transactions: []core.Transaction{fakes.FakeTransaction}, } - blockRepository.CreateOrUpdateBlock(block) + _, insertErr := blockRepository.CreateOrUpdateBlock(block) - savedBlock, _ := blockRepository.GetBlock(123) + Expect(insertErr).NotTo(HaveOccurred()) + savedBlock, getErr := blockRepository.GetBlock(123) + Expect(getErr).NotTo(HaveOccurred()) Expect(len(savedBlock.Transactions)).To(Equal(1)) }) It("saves two transactions associated to the block", func() { block := core.Block{ Number: 123, - Transactions: []core.Transaction{{}, {}}, + Transactions: []core.Transaction{fakes.FakeTransaction, fakes.FakeTransaction}, } - blockRepository.CreateOrUpdateBlock(block) + _, insertErr := blockRepository.CreateOrUpdateBlock(block) - savedBlock, _ := blockRepository.GetBlock(123) + Expect(insertErr).NotTo(HaveOccurred()) + savedBlock, getErr := blockRepository.GetBlock(123) + Expect(getErr).NotTo(HaveOccurred()) Expect(len(savedBlock.Transactions)).To(Equal(2)) }) It(`replaces blocks and transactions associated to the block when a more new block is in conflict (same block number + nodeid)`, func() { blockOne := core.Block{ - Number: 123, - Hash: "xabc", - Transactions: []core.Transaction{{Hash: "x123"}, {Hash: "x345"}}, + Number: 123, + Hash: "xabc", + Transactions: []core.Transaction{ + fakes.GetFakeTransaction("x123", core.Receipt{}), + fakes.GetFakeTransaction("x345", core.Receipt{}), + }, } blockTwo := core.Block{ - Number: 123, - Hash: "xdef", - Transactions: []core.Transaction{{Hash: "x678"}, {Hash: "x9ab"}}, + Number: 123, + Hash: "xdef", + Transactions: []core.Transaction{ + fakes.GetFakeTransaction("x678", core.Receipt{}), + fakes.GetFakeTransaction("x9ab", core.Receipt{}), + }, } - blockRepository.CreateOrUpdateBlock(blockOne) - blockRepository.CreateOrUpdateBlock(blockTwo) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(blockOne) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(blockTwo) + Expect(insertErrTwo).NotTo(HaveOccurred()) savedBlock, _ := blockRepository.GetBlock(123) Expect(len(savedBlock.Transactions)).To(Equal(2)) @@ -173,14 +191,21 @@ var _ = Describe("Saving blocks", func() { It(`does not replace blocks when block number is not unique but block number + node id is`, func() { blockOne := core.Block{ - Number: 123, - Transactions: []core.Transaction{{Hash: "x123"}, {Hash: "x345"}}, + Number: 123, + Transactions: []core.Transaction{ + fakes.GetFakeTransaction("x123", core.Receipt{}), + fakes.GetFakeTransaction("x345", core.Receipt{}), + }, } blockTwo := core.Block{ - Number: 123, - Transactions: []core.Transaction{{Hash: "x678"}, {Hash: "x9ab"}}, + Number: 123, + Transactions: []core.Transaction{ + fakes.GetFakeTransaction("x678", core.Receipt{}), + fakes.GetFakeTransaction("x9ab", core.Receipt{}), + }, } - blockRepository.CreateOrUpdateBlock(blockOne) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(blockOne) + Expect(insertErrOne).NotTo(HaveOccurred()) nodeTwo := core.Node{ GenesisBlock: "0x456", NetworkID: 1, @@ -189,10 +214,14 @@ var _ = Describe("Saving blocks", func() { test_config.CleanTestDB(dbTwo) repositoryTwo := repositories.NewBlockRepository(dbTwo) - blockRepository.CreateOrUpdateBlock(blockOne) - repositoryTwo.CreateOrUpdateBlock(blockTwo) - retrievedBlockOne, _ := blockRepository.GetBlock(123) - retrievedBlockTwo, _ := repositoryTwo.GetBlock(123) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(blockOne) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := repositoryTwo.CreateOrUpdateBlock(blockTwo) + Expect(insertErrThree).NotTo(HaveOccurred()) + retrievedBlockOne, getErrOne := blockRepository.GetBlock(123) + Expect(getErrOne).NotTo(HaveOccurred()) + retrievedBlockTwo, getErrTwo := repositoryTwo.GetBlock(123) + Expect(getErrTwo).NotTo(HaveOccurred()) Expect(retrievedBlockOne.Transactions[0].Hash).To(Equal("x123")) Expect(retrievedBlockTwo.Transactions[0].Hash).To(Equal("x678")) @@ -223,46 +252,51 @@ var _ = Describe("Saving blocks", func() { var value = new(big.Int) value.SetString("34940183920000000000", 10) inputData := "0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14" + gethTransaction := types.NewTransaction(nonce, common.HexToAddress(to), value, gasLimit, big.NewInt(gasPrice), common.FromHex(inputData)) + var raw bytes.Buffer + rlpErr := gethTransaction.EncodeRLP(&raw) + Expect(rlpErr).NotTo(HaveOccurred()) transaction := core.Transaction{ - Hash: "x1234", - GasPrice: gasPrice, - GasLimit: gasLimit, - Nonce: nonce, - To: to, - From: from, - Value: value.String(), Data: inputData, + From: from, + GasLimit: gasLimit, + GasPrice: gasPrice, + Hash: "x1234", + Nonce: nonce, + Raw: raw.Bytes(), + Receipt: core.Receipt{}, + To: to, + TxIndex: 2, + Value: value.String(), } block := core.Block{ Number: 123, Transactions: []core.Transaction{transaction}, } - blockRepository.CreateOrUpdateBlock(block) + _, insertErr := blockRepository.CreateOrUpdateBlock(block) + Expect(insertErr).NotTo(HaveOccurred()) - savedBlock, _ := blockRepository.GetBlock(123) + savedBlock, err := blockRepository.GetBlock(123) + Expect(err).NotTo(HaveOccurred()) Expect(len(savedBlock.Transactions)).To(Equal(1)) savedTransaction := savedBlock.Transactions[0] - Expect(savedTransaction.Data).To(Equal(transaction.Data)) - Expect(savedTransaction.Hash).To(Equal(transaction.Hash)) - Expect(savedTransaction.To).To(Equal(to)) - Expect(savedTransaction.From).To(Equal(from)) - Expect(savedTransaction.Nonce).To(Equal(nonce)) - Expect(savedTransaction.GasLimit).To(Equal(gasLimit)) - Expect(savedTransaction.GasPrice).To(Equal(gasPrice)) - Expect(savedTransaction.Value).To(Equal(value.String())) + Expect(savedTransaction).To(Equal(transaction)) }) Describe("The missing block numbers", func() { It("is empty the starting block number is the highest known block number", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 1}) + _, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 1}) + Expect(insertErr).NotTo(HaveOccurred()) Expect(len(blockRepository.MissingBlockNumbers(1, 1, node.ID))).To(Equal(0)) }) It("is empty if copies of block exist from both current node and another", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 0}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 1}) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 0}) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 1}) + Expect(insertErrTwo).NotTo(HaveOccurred()) nodeTwo := core.Node{ GenesisBlock: "0x456", NetworkID: 1, @@ -270,7 +304,8 @@ var _ = Describe("Saving blocks", func() { dbTwo, err := postgres.NewDB(test_config.DBConfig, nodeTwo) Expect(err).NotTo(HaveOccurred()) repositoryTwo := repositories.NewBlockRepository(dbTwo) - repositoryTwo.CreateOrUpdateBlock(core.Block{Number: 0}) + _, insertErrThree := repositoryTwo.CreateOrUpdateBlock(core.Block{Number: 0}) + Expect(insertErrThree).NotTo(HaveOccurred()) missing := blockRepository.MissingBlockNumbers(0, 1, node.ID) @@ -278,42 +313,53 @@ var _ = Describe("Saving blocks", func() { }) It("is the only missing block number", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 2}) + _, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 2}) + Expect(insertErr).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(1, 2, node.ID)).To(Equal([]int64{1})) }) It("is both missing block numbers", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) + _, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) + Expect(insertErr).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(1, 3, node.ID)).To(Equal([]int64{1, 2})) }) It("goes back to the starting block number", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + _, insertErr := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + Expect(insertErr).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{4, 5})) }) It("only includes missing block numbers", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + Expect(insertErrTwo).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(4, 6, node.ID)).To(Equal([]int64{5})) }) It("includes blocks created by a different node", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 6}) + Expect(insertErrTwo).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(4, 6, "Different node id")).To(Equal([]int64{4, 5, 6})) }) It("is a list with multiple gaps", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 5}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 8}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 10}) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 4}) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 5}) + Expect(insertErrTwo).NotTo(HaveOccurred()) + _, insertErrThree := blockRepository.CreateOrUpdateBlock(core.Block{Number: 8}) + Expect(insertErrThree).NotTo(HaveOccurred()) + _, insertErrFour := blockRepository.CreateOrUpdateBlock(core.Block{Number: 10}) + Expect(insertErrFour).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(3, 10, node.ID)).To(Equal([]int64{3, 6, 7, 9})) }) @@ -323,8 +369,10 @@ var _ = Describe("Saving blocks", func() { }) It("only returns requested range even when other gaps exist", func() { - blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) - blockRepository.CreateOrUpdateBlock(core.Block{Number: 8}) + _, insertErrOne := blockRepository.CreateOrUpdateBlock(core.Block{Number: 3}) + Expect(insertErrOne).NotTo(HaveOccurred()) + _, insertErrTwo := blockRepository.CreateOrUpdateBlock(core.Block{Number: 8}) + Expect(insertErrTwo).NotTo(HaveOccurred()) Expect(blockRepository.MissingBlockNumbers(1, 5, node.ID)).To(Equal([]int64{1, 2, 4, 5})) }) @@ -334,11 +382,13 @@ var _ = Describe("Saving blocks", func() { It("sets the status of blocks within n-20 of chain HEAD as final", func() { blockNumberOfChainHead := 25 for i := 0; i < blockNumberOfChainHead; i++ { - blockRepository.CreateOrUpdateBlock(core.Block{Number: int64(i), Hash: strconv.Itoa(i)}) + _, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: int64(i), Hash: strconv.Itoa(i)}) + Expect(err).NotTo(HaveOccurred()) } - blockRepository.SetBlocksStatus(int64(blockNumberOfChainHead)) + setErr := blockRepository.SetBlocksStatus(int64(blockNumberOfChainHead)) + Expect(setErr).NotTo(HaveOccurred()) blockOne, err := blockRepository.GetBlock(1) Expect(err).ToNot(HaveOccurred()) Expect(blockOne.IsFinal).To(Equal(true)) diff --git a/pkg/datastore/postgres/repositories/contract_repository.go b/pkg/datastore/postgres/repositories/contract_repository.go index 317f4d28..fc68f510 100644 --- a/pkg/datastore/postgres/repositories/contract_repository.go +++ b/pkg/datastore/postgres/repositories/contract_repository.go @@ -42,7 +42,7 @@ func (contractRepository ContractRepository) CreateContract(contract core.Contra SET contract_hash = $1, contract_abi = $2 `, contract.Hash, abiToInsert) if err != nil { - return postgres.ErrDBInsertFailed + return postgres.ErrDBInsertFailed(err) } return nil } @@ -86,7 +86,7 @@ func (contractRepository ContractRepository) addTransactions(contract core.Contr gasprice, value, input_data - FROM transactions + FROM full_sync_transactions WHERE tx_to = $1 ORDER BY block_id DESC`, contract.Hash) if err != nil { diff --git a/pkg/datastore/postgres/repositories/logs_repository.go b/pkg/datastore/postgres/repositories/logs_repository.go index 51da7a7d..4c13e723 100644 --- a/pkg/datastore/postgres/repositories/logs_repository.go +++ b/pkg/datastore/postgres/repositories/logs_repository.go @@ -32,18 +32,18 @@ type LogRepository struct { func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error { tx, _ := logRepository.DB.Beginx() for _, tlog := range lgs { - _, err := tx.Exec( + _, insertLogErr := tx.Exec( `INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `, tlog.BlockNumber, tlog.Address, tlog.TxHash, tlog.Index, tlog.Topics[0], tlog.Topics[1], tlog.Topics[2], tlog.Topics[3], tlog.Data, receiptId, ) - if err != nil { - err = tx.Rollback() - if err != nil { - logrus.Error("CreateLogs: could not perform rollback: ", err) + if insertLogErr != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + logrus.Error("CreateLogs: could not perform rollback: ", rollbackErr) } - return postgres.ErrDBInsertFailed + return postgres.ErrDBInsertFailed(insertLogErr) } } err := tx.Commit() @@ -52,7 +52,7 @@ func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) e if err != nil { logrus.Error("CreateLogs: could not perform rollback: ", err) } - return postgres.ErrDBInsertFailed + return postgres.ErrDBInsertFailed(err) } return nil } diff --git a/pkg/datastore/postgres/repositories/logs_repository_test.go b/pkg/datastore/postgres/repositories/logs_repository_test.go index 2e4c843d..7cf319a4 100644 --- a/pkg/datastore/postgres/repositories/logs_repository_test.go +++ b/pkg/datastore/postgres/repositories/logs_repository_test.go @@ -17,6 +17,7 @@ package repositories_test import ( + "github.com/vulcanize/vulcanizedb/pkg/fakes" "sort" . "github.com/onsi/ginkgo" @@ -201,11 +202,7 @@ var _ = Describe("Logs Repository", func() { Status: 1, TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", } - transaction := - core.Transaction{ - Hash: receipt.TxHash, - Receipt: receipt, - } + transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt) block := core.Block{Transactions: []core.Transaction{transaction}} _, err := blockRepository.CreateOrUpdateBlock(block) diff --git a/pkg/datastore/postgres/repositories/receipts_repository_test.go b/pkg/datastore/postgres/repositories/receipts_repository_test.go index 073d7531..b2e4433a 100644 --- a/pkg/datastore/postgres/repositories/receipts_repository_test.go +++ b/pkg/datastore/postgres/repositories/receipts_repository_test.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -117,10 +118,7 @@ var _ = Describe("Receipts Repository", func() { TxHash: "0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223", } - transaction := core.Transaction{ - Hash: expected.TxHash, - Receipt: expected, - } + transaction := fakes.GetFakeTransaction(expected.TxHash, expected) block := core.Block{Transactions: []core.Transaction{transaction}} _, err := blockRepository.CreateOrUpdateBlock(block) @@ -147,10 +145,7 @@ var _ = Describe("Receipts Repository", func() { receipt := core.Receipt{ TxHash: "0x002c4799161d809b23f67884eb6598c9df5894929fe1a9ead97ca175d360f547", } - transaction := core.Transaction{ - Hash: receipt.TxHash, - Receipt: receipt, - } + transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt) block := core.Block{ Transactions: []core.Transaction{transaction}, diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go index 07f6abe3..eeb25fe9 100644 --- a/pkg/fakes/data.go +++ b/pkg/fakes/data.go @@ -17,6 +17,7 @@ package fakes import ( + "bytes" "encoding/json" "errors" "strconv" @@ -48,3 +49,42 @@ func GetFakeHeader(blockNumber int64) core.Header { Timestamp: strconv.FormatInt(fakeTimestamp, 10), } } + +var fakeTransaction types.Transaction +var rawTransaction bytes.Buffer +var _ = fakeTransaction.EncodeRLP(&rawTransaction) +var FakeTransaction = core.Transaction{ + Data: "", + From: "", + GasLimit: 0, + GasPrice: 0, + Hash: "", + Nonce: 0, + Raw: rawTransaction.Bytes(), + Receipt: core.Receipt{}, + To: "", + TxIndex: 0, + Value: "0", +} + +func GetFakeTransaction(hash string, receipt core.Receipt) core.Transaction { + gethTransaction := types.Transaction{} + var raw bytes.Buffer + err := gethTransaction.EncodeRLP(&raw) + if err != nil { + panic("failed to marshal transaction creating test fake") + } + return core.Transaction{ + Data: "", + From: "", + GasLimit: 0, + GasPrice: 0, + Hash: hash, + Nonce: 0, + Raw: raw.Bytes(), + Receipt: receipt, + To: "", + TxIndex: 0, + Value: "0", + } +} diff --git a/pkg/geth/converters/common/block_converter_test.go b/pkg/geth/converters/common/block_converter_test.go index 7e73967e..cf4ac8d8 100644 --- a/pkg/geth/converters/common/block_converter_test.go +++ b/pkg/geth/converters/common/block_converter_test.go @@ -17,6 +17,7 @@ package common_test import ( + "bytes" "io/ioutil" "log" "math/big" @@ -227,6 +228,9 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { big.NewInt(3), hexutil.MustDecode("0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14"), ) + var rawTransaction bytes.Buffer + encodeErr := gethTransaction.EncodeRLP(&rawTransaction) + Expect(encodeErr).NotTo(HaveOccurred()) gethReceipt := &types.Receipt{ Bloom: types.BytesToBloom(hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")), @@ -261,6 +265,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123")) Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas())) Expect(coreTransaction.GasPrice).To(Equal(gethTransaction.GasPrice().Int64())) + Expect(coreTransaction.Raw).To(Equal(rawTransaction.Bytes())) + Expect(coreTransaction.TxIndex).To(Equal(int64(0))) Expect(coreTransaction.Value).To(Equal(gethTransaction.Value().String())) Expect(coreTransaction.Nonce).To(Equal(gethTransaction.Nonce())) diff --git a/pkg/geth/converters/rpc/transaction_converter.go b/pkg/geth/converters/rpc/transaction_converter.go index ebec4c43..78f6e9c6 100644 --- a/pkg/geth/converters/rpc/transaction_converter.go +++ b/pkg/geth/converters/rpc/transaction_converter.go @@ -17,6 +17,7 @@ package rpc import ( + "bytes" "context" "log" "strings" @@ -52,7 +53,10 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B log.Println("transaction sender: ", err) return err } - coreTransaction := transToCoreTrans(transaction, &from) + coreTransaction, convertErr := transToCoreTrans(transaction, &from, int64(gethTransactionIndex)) + if convertErr != nil { + return convertErr + } coreTransaction, err = rtc.appendReceiptToTransaction(coreTransaction) if err != nil { log.Println("receipt: ", err) @@ -79,18 +83,25 @@ func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core. return transaction, nil } -func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.Transaction { +func transToCoreTrans(transaction *types.Transaction, from *common.Address, transactionIndex int64) (core.Transaction, error) { data := hexutil.Encode(transaction.Data()) + var raw bytes.Buffer + encodeErr := transaction.EncodeRLP(&raw) + if encodeErr != nil { + return core.Transaction{}, encodeErr + } return core.Transaction{ - Hash: transaction.Hash().Hex(), - Nonce: transaction.Nonce(), - To: strings.ToLower(addressToHex(transaction.To())), + Data: data, From: strings.ToLower(addressToHex(from)), GasLimit: transaction.Gas(), GasPrice: transaction.GasPrice().Int64(), + Hash: transaction.Hash().Hex(), + Nonce: transaction.Nonce(), + Raw: raw.Bytes(), + To: strings.ToLower(addressToHex(transaction.To())), + TxIndex: transactionIndex, Value: transaction.Value().String(), - Data: data, - } + }, nil } func addressToHex(to *common.Address) string { diff --git a/pkg/plugin/test_helpers/database.go b/pkg/plugin/test_helpers/database.go index 345683f0..6fa82ad7 100644 --- a/pkg/plugin/test_helpers/database.go +++ b/pkg/plugin/test_helpers/database.go @@ -66,7 +66,7 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM log_filters`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DELETE FROM transactions`) + _, err = tx.Exec(`DELETE FROM full_sync_transactions`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM receipts`) diff --git a/test_config/test_config.go b/test_config/test_config.go index 0f0071ae..60a131f9 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -108,13 +108,13 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM blocks") db.MustExec("DELETE FROM checked_headers") // can't delete from eth_nodes since this function is called after the required eth_node is persisted + db.MustExec("DELETE FROM full_sync_transactions") db.MustExec("DELETE FROM goose_db_version") db.MustExec("DELETE FROM headers") db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") db.MustExec("DELETE FROM queued_storage") db.MustExec("DELETE FROM receipts") - db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts") } From 79e011aad2d2287b52ddcf77c9fda0abe7da28cb Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 19 Mar 2019 14:39:26 -0500 Subject: [PATCH 2/5] Add light sync transactions table - Foreign key to header instead of block - Can use same Transaction struct for both --- ...5_create_light_sync_transactions_table.sql | 14 ++++ db/schema.sql | 66 +++++++++++++++++++ .../shared/helpers/test_helpers/database.go | 3 + .../repositories/header_repository.go | 12 +++- .../repositories/header_repository_test.go | 51 ++++++++++++++ pkg/datastore/repository.go | 1 + pkg/fakes/mock_header_repository.go | 4 ++ pkg/plugin/test_helpers/database.go | 3 + test_config/test_config.go | 1 + 9 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 db/migrations/00025_create_light_sync_transactions_table.sql diff --git a/db/migrations/00025_create_light_sync_transactions_table.sql b/db/migrations/00025_create_light_sync_transactions_table.sql new file mode 100644 index 00000000..f72f503e --- /dev/null +++ b/db/migrations/00025_create_light_sync_transactions_table.sql @@ -0,0 +1,14 @@ +-- +goose Up +CREATE TABLE light_sync_transactions ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE, + hash TEXT, + raw JSONB, + tx_index INTEGER, + tx_from TEXT, + tx_to TEXT, + UNIQUE (header_id, hash) +); + +-- +goose Down +DROP TABLE light_sync_transactions; diff --git a/db/schema.sql b/db/schema.sql index d6a05de6..d5496079 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -258,6 +258,41 @@ CREATE SEQUENCE public.headers_id_seq ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id; +-- +-- Name: light_sync_transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.light_sync_transactions ( + id integer NOT NULL, + header_id integer NOT NULL, + hash text, + raw jsonb, + tx_index integer, + tx_from text, + tx_to text +); + + +-- +-- Name: light_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.light_sync_transactions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: light_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.light_sync_transactions_id_seq OWNED BY public.light_sync_transactions.id; + + -- -- Name: log_filters; Type: TABLE; Schema: public; Owner: - -- @@ -504,6 +539,13 @@ ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('pu ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass); +-- +-- Name: light_sync_transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.light_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.light_sync_transactions_id_seq'::regclass); + + -- -- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: - -- @@ -603,6 +645,22 @@ ALTER TABLE ONLY public.headers ADD CONSTRAINT headers_pkey PRIMARY KEY (id); +-- +-- Name: light_sync_transactions light_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.light_sync_transactions + ADD CONSTRAINT light_sync_transactions_header_id_hash_key UNIQUE (header_id, hash); + + +-- +-- Name: light_sync_transactions light_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.light_sync_transactions + ADD CONSTRAINT light_sync_transactions_pkey PRIMARY KEY (id); + + -- -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -733,6 +791,14 @@ ALTER TABLE ONLY public.full_sync_transactions ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE; +-- +-- Name: light_sync_transactions light_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.light_sync_transactions + ADD CONSTRAINT light_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index c33a304e..8459c21c 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -240,6 +240,9 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM full_sync_transactions`) Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec("DELETE FROM light_sync_transactions") + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM receipts`) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 83126e1f..1f886003 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -49,11 +49,21 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int return 0, ErrValidHeaderExists } +func (repository HeaderRepository) CreateTransaction(headerID int64, transaction core.Transaction) error { + _, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions + (header_id, hash, tx_to, tx_from, tx_index) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT DO NOTHING`, + headerID, transaction.Hash, transaction.To, transaction.From, transaction.TxIndex) + return err +} + func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { var header core.Header err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`, blockNumber, repository.database.Node.ID) - log.Error("GetHeader: error getting headers: ", err) + if err != nil { + log.Error("GetHeader: error getting headers: ", err) + } return header, err } diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index 17dd5dbb..d2dbd33c 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -180,6 +180,57 @@ var _ = Describe("Block header repository", func() { }) }) + Describe("creating a transaction", func() { + It("adds a transaction", func() { + headerID, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + fromAddress := common.HexToAddress("0x1234") + toAddress := common.HexToAddress("0x5678") + txHash := common.HexToHash("0x9876") + txIndex := big.NewInt(123) + transaction := core.Transaction{ + From: fromAddress.Hex(), + Hash: txHash.Hex(), + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + } + + insertErr := repo.CreateTransaction(headerID, transaction) + + Expect(insertErr).NotTo(HaveOccurred()) + var dbTransaction core.Transaction + err = db.Get(&dbTransaction, `SELECT hash, tx_from, tx_to, tx_index FROM public.light_sync_transactions WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbTransaction).To(Equal(transaction)) + }) + + It("silently ignores duplicate inserts", func() { + headerID, err := repo.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + fromAddress := common.HexToAddress("0x1234") + toAddress := common.HexToAddress("0x5678") + txHash := common.HexToHash("0x9876") + txIndex := big.NewInt(123) + transaction := core.Transaction{ + From: fromAddress.Hex(), + Hash: txHash.Hex(), + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + } + + insertErr := repo.CreateTransaction(headerID, transaction) + Expect(insertErr).NotTo(HaveOccurred()) + + insertTwoErr := repo.CreateTransaction(headerID, transaction) + Expect(insertTwoErr).NotTo(HaveOccurred()) + + var dbTransactions []core.Transaction + err = db.Select(&dbTransactions, `SELECT hash, tx_from, tx_to, tx_index FROM public.light_sync_transactions WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbTransactions)).To(Equal(1)) + }) + }) + Describe("Getting a header", func() { It("returns header if it exists", func() { _, err = repo.CreateOrUpdateHeader(header) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index 7f942051..b26bb8ce 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -55,6 +55,7 @@ type FilterRepository interface { type HeaderRepository interface { CreateOrUpdateHeader(header core.Header) (int64, error) + CreateTransaction(headerID int64, transaction core.Transaction) error GetHeader(blockNumber int64) (core.Header, error) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) } diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index 42a21f5c..ef196949 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -56,6 +56,10 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr } +func (repository *MockHeaderRepository) CreateTransaction(headerID int64, transaction core.Transaction) error { + panic("implement me") +} + func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { repository.GetHeaderPassedBlockNumber = blockNumber return core.Header{BlockNumber: blockNumber, Hash: repository.getHeaderReturnBlockHash}, repository.getHeaderError diff --git a/pkg/plugin/test_helpers/database.go b/pkg/plugin/test_helpers/database.go index 6fa82ad7..e4b941af 100644 --- a/pkg/plugin/test_helpers/database.go +++ b/pkg/plugin/test_helpers/database.go @@ -69,6 +69,9 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`DELETE FROM full_sync_transactions`) Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec("DELETE FROM light_sync_transactions") + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM receipts`) Expect(err).NotTo(HaveOccurred()) diff --git a/test_config/test_config.go b/test_config/test_config.go index 60a131f9..1e2fbdc3 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -111,6 +111,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM full_sync_transactions") db.MustExec("DELETE FROM goose_db_version") db.MustExec("DELETE FROM headers") + db.MustExec("DELETE FROM light_sync_transactions") db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") db.MustExec("DELETE FROM queued_storage") From 54d46638a8bf789db0ad6a53828cc7338079ea73 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 26 Mar 2019 23:05:30 -0500 Subject: [PATCH 3/5] updates for light sync transactions --- ...02_create_full_sync_transactions_table.sql | 2 +- ...5_create_light_sync_transactions_table.sql | 19 ++- db/schema.sql | 13 +- libraries/shared/transactions/syncer.go | 55 ++++++ libraries/shared/transactions/syncer_test.go | 80 +++++++++ .../transactions/transactions_suite_test.go | 13 ++ libraries/shared/watcher/event_watcher.go | 76 ++++++--- .../shared/watcher/event_watcher_test.go | 27 +++ .../full/retriever/block_retriever_test.go | 18 +- .../helpers/test_helpers/mocks/entities.go | 8 +- pkg/core/block.go | 2 +- pkg/core/blockchain.go | 4 +- pkg/core/contract.go | 2 +- pkg/core/transaction.go | 19 ++- pkg/datastore/errors.go | 19 +++ pkg/datastore/postgres/postgres_test.go | 6 +- .../postgres/repositories/block_repository.go | 14 +- .../repositories/block_repository_test.go | 18 +- .../repositories/contract_repository_test.go | 14 +- .../repositories/header_repository.go | 10 +- .../repositories/header_repository_test.go | 45 +++-- .../repositories/logs_repository_test.go | 2 +- ...ory_test.go => receipt_repository_test.go} | 6 +- pkg/datastore/repository.go | 20 +-- pkg/fakes/data.go | 11 +- pkg/fakes/mock_blockchain.go | 11 ++ pkg/fakes/mock_header_repository.go | 7 +- pkg/fakes/mock_transaction_converter.go | 35 ++-- pkg/fakes/mock_transaction_syncer.go | 13 ++ pkg/geth/blockchain.go | 160 ++++++++++-------- pkg/geth/blockchain_test.go | 40 +++-- pkg/geth/cold_import/importer_test.go | 3 +- .../cold_db/transaction_converter.go | 16 +- pkg/geth/converters/common/block_converter.go | 2 +- .../converters/common/block_converter_test.go | 3 +- .../common/transaction_converter.go | 3 +- pkg/geth/converters/rpc/rpc_suite_test.go | 13 ++ .../converters/rpc/transaction_converter.go | 134 +++++++++++++-- .../rpc/transaction_converter_test.go | 82 +++++++++ 39 files changed, 769 insertions(+), 256 deletions(-) create mode 100644 libraries/shared/transactions/syncer.go create mode 100644 libraries/shared/transactions/syncer_test.go create mode 100644 libraries/shared/transactions/transactions_suite_test.go create mode 100644 pkg/datastore/errors.go rename pkg/datastore/postgres/repositories/{receipts_repository_test.go => receipt_repository_test.go} (96%) create mode 100644 pkg/fakes/mock_transaction_syncer.go create mode 100644 pkg/geth/converters/rpc/rpc_suite_test.go create mode 100644 pkg/geth/converters/rpc/transaction_converter_test.go diff --git a/db/migrations/00002_create_full_sync_transactions_table.sql b/db/migrations/00002_create_full_sync_transactions_table.sql index f1e0be23..dc42a7c7 100644 --- a/db/migrations/00002_create_full_sync_transactions_table.sql +++ b/db/migrations/00002_create_full_sync_transactions_table.sql @@ -5,7 +5,7 @@ CREATE TABLE full_sync_transactions ( gaslimit NUMERIC, gasprice NUMERIC, hash VARCHAR(66), - input_data VARCHAR, + input_data BYTEA, nonce NUMERIC, raw BYTEA, tx_from VARCHAR(66), diff --git a/db/migrations/00025_create_light_sync_transactions_table.sql b/db/migrations/00025_create_light_sync_transactions_table.sql index f72f503e..6773c1e3 100644 --- a/db/migrations/00025_create_light_sync_transactions_table.sql +++ b/db/migrations/00025_create_light_sync_transactions_table.sql @@ -1,12 +1,17 @@ -- +goose Up CREATE TABLE light_sync_transactions ( - id SERIAL PRIMARY KEY, - header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE, - hash TEXT, - raw JSONB, - tx_index INTEGER, - tx_from TEXT, - tx_to TEXT, + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE, + hash TEXT, + gaslimit NUMERIC, + gasprice NUMERIC, + input_data BYTEA, + nonce NUMERIC, + raw BYTEA, + tx_from TEXT, + tx_index INTEGER, + tx_to TEXT, + "value" NUMERIC, UNIQUE (header_id, hash) ); diff --git a/db/schema.sql b/db/schema.sql index d5496079..bff9d609 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -161,7 +161,7 @@ CREATE TABLE public.full_sync_transactions ( gaslimit numeric, gasprice numeric, hash character varying(66), - input_data character varying, + input_data bytea, nonce numeric, raw bytea, tx_from character varying(66), @@ -266,10 +266,15 @@ CREATE TABLE public.light_sync_transactions ( id integer NOT NULL, header_id integer NOT NULL, hash text, - raw jsonb, - tx_index integer, + gaslimit numeric, + gasprice numeric, + input_data bytea, + nonce numeric, + raw bytea, tx_from text, - tx_to text + tx_index integer, + tx_to text, + value numeric ); diff --git a/libraries/shared/transactions/syncer.go b/libraries/shared/transactions/syncer.go new file mode 100644 index 00000000..5f3085c7 --- /dev/null +++ b/libraries/shared/transactions/syncer.go @@ -0,0 +1,55 @@ +package transactions + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" +) + +type ITransactionsSyncer interface { + SyncTransactions(headerID int64, logs []types.Log) error +} + +type TransactionsSyncer struct { + BlockChain core.BlockChain + Repository datastore.HeaderRepository +} + +func NewTransactionsSyncer(db *postgres.DB, blockChain core.BlockChain) TransactionsSyncer { + repository := repositories.NewHeaderRepository(db) + return TransactionsSyncer{ + BlockChain: blockChain, + Repository: repository, + } +} + +func (syncer TransactionsSyncer) SyncTransactions(headerID int64, logs []types.Log) error { + transactionHashes := getUniqueTransactionHashes(logs) + transactions, transactionErr := syncer.BlockChain.GetTransactions(transactionHashes) + if transactionErr != nil { + return transactionErr + } + for _, transaction := range transactions { + writeErr := syncer.Repository.CreateTransaction(headerID, transaction) + if writeErr != nil { + return writeErr + } + } + return nil +} + +func getUniqueTransactionHashes(logs []types.Log) []common.Hash { + seen := make(map[common.Hash]struct{}, len(logs)) + var result []common.Hash + for _, log := range logs { + if _, ok := seen[log.TxHash]; ok { + continue + } + seen[log.TxHash] = struct{}{} + result = append(result, log.TxHash) + } + return result +} diff --git a/libraries/shared/transactions/syncer_test.go b/libraries/shared/transactions/syncer_test.go new file mode 100644 index 00000000..954d2a0b --- /dev/null +++ b/libraries/shared/transactions/syncer_test.go @@ -0,0 +1,80 @@ +package transactions_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/libraries/shared/transactions" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Transaction syncer", func() { + It("fetches transactions for logs", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + blockChain := fakes.NewMockBlockChain() + syncer := transactions.NewTransactionsSyncer(db, blockChain) + + err := syncer.SyncTransactions(0, []types.Log{}) + + Expect(err).NotTo(HaveOccurred()) + Expect(blockChain.GetTransactionsCalled).To(BeTrue()) + }) + + It("only fetches transactions with unique hashes", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + blockChain := fakes.NewMockBlockChain() + syncer := transactions.NewTransactionsSyncer(db, blockChain) + + err := syncer.SyncTransactions(0, []types.Log{{ + TxHash: fakes.FakeHash, + }, { + TxHash: fakes.FakeHash, + }}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(blockChain.GetTransactionsPassedHashes)).To(Equal(1)) + }) + + It("returns error if fetching transactions fails", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + blockChain := fakes.NewMockBlockChain() + blockChain.GetTransactionsError = fakes.FakeError + syncer := transactions.NewTransactionsSyncer(db, blockChain) + + err := syncer.SyncTransactions(0, []types.Log{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("passes transactions to repository for persistence", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + blockChain := fakes.NewMockBlockChain() + blockChain.Transactions = []core.TransactionModel{{}} + syncer := transactions.NewTransactionsSyncer(db, blockChain) + mockHeaderRepository := fakes.NewMockHeaderRepository() + syncer.Repository = mockHeaderRepository + + err := syncer.SyncTransactions(0, []types.Log{}) + + Expect(err).NotTo(HaveOccurred()) + Expect(mockHeaderRepository.CreateTransactionCalled).To(BeTrue()) + }) + + It("returns error if persisting transactions fails", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + blockChain := fakes.NewMockBlockChain() + blockChain.Transactions = []core.TransactionModel{{}} + syncer := transactions.NewTransactionsSyncer(db, blockChain) + mockHeaderRepository := fakes.NewMockHeaderRepository() + mockHeaderRepository.CreateTransactionError = fakes.FakeError + syncer.Repository = mockHeaderRepository + + err := syncer.SyncTransactions(0, []types.Log{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/libraries/shared/transactions/transactions_suite_test.go b/libraries/shared/transactions/transactions_suite_test.go new file mode 100644 index 00000000..1771a687 --- /dev/null +++ b/libraries/shared/transactions/transactions_suite_test.go @@ -0,0 +1,13 @@ +package transactions_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestTransactions(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Shared Transactions Suite") +} diff --git a/libraries/shared/watcher/event_watcher.go b/libraries/shared/watcher/event_watcher.go index cdb2561f..9bad688a 100644 --- a/libraries/shared/watcher/event_watcher.go +++ b/libraries/shared/watcher/event_watcher.go @@ -18,14 +18,16 @@ package watcher import ( "fmt" + "github.com/vulcanize/vulcanizedb/libraries/shared/transactions" "github.com/ethereum/go-ethereum/common" - log "github.com/sirupsen/logrus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/sirupsen/logrus" - chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker" + "github.com/vulcanize/vulcanizedb/libraries/shared/chunker" "github.com/vulcanize/vulcanizedb/libraries/shared/constants" - fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" - repo "github.com/vulcanize/vulcanizedb/libraries/shared/repository" + "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" + "github.com/vulcanize/vulcanizedb/libraries/shared/repository" "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -33,21 +35,26 @@ import ( type EventWatcher struct { Transformers []transformer.EventTransformer + BlockChain core.BlockChain DB *postgres.DB - Fetcher fetch.LogFetcher - Chunker chunk.Chunker + Fetcher fetcher.LogFetcher + Chunker chunker.Chunker Addresses []common.Address Topics []common.Hash StartingBlock *int64 + Syncer transactions.ITransactionsSyncer } func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher { - chunker := chunk.NewLogChunker() - fetcher := fetch.NewFetcher(bc) + logChunker := chunker.NewLogChunker() + logFetcher := fetcher.NewFetcher(bc) + transactionSyncer := transactions.NewTransactionsSyncer(db, bc) return EventWatcher{ - DB: db, - Fetcher: fetcher, - Chunker: chunker, + BlockChain: bc, + DB: db, + Fetcher: logFetcher, + Chunker: logChunker, + Syncer: transactionSyncer, } } @@ -85,15 +92,15 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti return fmt.Errorf("No transformers added to watcher") } - checkedColumnNames, err := repo.GetCheckedColumnNames(watcher.DB) + checkedColumnNames, err := repository.GetCheckedColumnNames(watcher.DB) if err != nil { return err } - notCheckedSQL := repo.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders) + notCheckedSQL := repository.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders) - missingHeaders, err := repo.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL) + missingHeaders, err := repository.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL) if err != nil { - log.Error("Fetching of missing headers failed in watcher!") + logrus.Error("Fetching of missing headers failed in watcher!") return err } @@ -101,28 +108,41 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti // TODO Extend FetchLogs for doing several blocks at a time logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) if err != nil { - // TODO Handle fetch error in watcher - log.Errorf("Error while fetching logs for header %v in watcher", header.Id) + logrus.Errorf("Error while fetching logs for header %v in watcher", header.Id) return err } - chunkedLogs := watcher.Chunker.ChunkLogs(logs) + transactionsSyncErr := watcher.Syncer.SyncTransactions(header.Id, logs) + if transactionsSyncErr != nil { + logrus.Errorf("error syncing transactions: %s", transactionsSyncErr.Error()) + return transactionsSyncErr + } - // Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync, - // not all logs we're interested in might have been fetched. - for _, t := range watcher.Transformers { - transformerName := t.GetConfig().TransformerName - logChunk := chunkedLogs[transformerName] - err = t.Execute(logChunk, header, constants.HeaderMissing) - if err != nil { - log.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err) - return err - } + transformErr := watcher.transformLogs(logs, header) + if transformErr != nil { + return transformErr } } return err } +func (watcher *EventWatcher) transformLogs(logs []types.Log, header core.Header) error { + chunkedLogs := watcher.Chunker.ChunkLogs(logs) + + // Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync, + // not all logs we're interested in might have been fetched. + for _, t := range watcher.Transformers { + transformerName := t.GetConfig().TransformerName + logChunk := chunkedLogs[transformerName] + err := t.Execute(logChunk, header, constants.HeaderMissing) + if err != nil { + logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err) + return err + } + } + return nil +} + func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool { return transformerBlock < watcherBlock } diff --git a/libraries/shared/watcher/event_watcher_test.go b/libraries/shared/watcher/event_watcher_test.go index 41b43372..6edd168d 100644 --- a/libraries/shared/watcher/event_watcher_test.go +++ b/libraries/shared/watcher/event_watcher_test.go @@ -121,6 +121,33 @@ var _ = Describe("Watcher", func() { w = watcher.NewEventWatcher(db, &mockBlockChain) }) + It("syncs transactions for fetched logs", func() { + fakeTransformer := &mocks.MockTransformer{} + w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer}) + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + mockTransactionSyncer := &fakes.MockTransactionSyncer{} + w.Syncer = mockTransactionSyncer + + err := w.Execute(constants.HeaderMissing) + + Expect(err).NotTo(HaveOccurred()) + Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue()) + }) + + It("returns error if syncing transactions fails", func() { + fakeTransformer := &mocks.MockTransformer{} + w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer}) + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + mockTransactionSyncer := &fakes.MockTransactionSyncer{} + mockTransactionSyncer.SyncTransactionsError = fakes.FakeError + w.Syncer = mockTransactionSyncer + + err := w.Execute(constants.HeaderMissing) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + It("executes each transformer", func() { fakeTransformer := &mocks.MockTransformer{} w.AddTransformers([]transformer.EventTransformerInitializer{fakeTransformer.FakeTransformerInitializer}) diff --git a/pkg/contract_watcher/full/retriever/block_retriever_test.go b/pkg/contract_watcher/full/retriever/block_retriever_test.go index 4b331983..f0255db0 100644 --- a/pkg/contract_watcher/full/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/full/retriever/block_retriever_test.go @@ -40,7 +40,7 @@ var _ = Describe("Block Retriever", func() { var block1 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 1, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } BeforeEach(func() { @@ -63,7 +63,7 @@ var _ = Describe("Block Retriever", func() { block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", GasPrice: 0, GasLimit: 0, @@ -83,7 +83,7 @@ var _ = Describe("Block Retriever", func() { block3 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", GasPrice: 0, GasLimit: 0, @@ -127,7 +127,7 @@ var _ = Describe("Block Retriever", func() { block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", GasPrice: 0, GasLimit: 0, @@ -158,7 +158,7 @@ var _ = Describe("Block Retriever", func() { block3 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", GasPrice: 0, GasLimit: 0, @@ -202,13 +202,13 @@ var _ = Describe("Block Retriever", func() { block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } block3 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) @@ -228,13 +228,13 @@ var _ = Describe("Block Retriever", func() { block2 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 2, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } block3 := core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", Number: 3, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } _, insertErrOne := blockRepository.CreateOrUpdateBlock(block1) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index 8bb37ce8..be6d5aec 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -33,7 +33,7 @@ import ( var TransferBlock1 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", Number: 6194633, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ GasLimit: 0, GasPrice: 0, Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", @@ -63,7 +63,7 @@ var TransferBlock1 = core.Block{ var TransferBlock2 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ooo", Number: 6194634, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ GasLimit: 0, GasPrice: 0, Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", @@ -93,7 +93,7 @@ var TransferBlock2 = core.Block{ var NewOwnerBlock1 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ppp", Number: 6194635, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ GasLimit: 0, GasPrice: 0, Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb", @@ -123,7 +123,7 @@ var NewOwnerBlock1 = core.Block{ var NewOwnerBlock2 = core.Block{ Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ggg", Number: 6194636, - Transactions: []core.Transaction{{ + Transactions: []core.TransactionModel{{ GasLimit: 0, GasPrice: 0, Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654lll", diff --git a/pkg/core/block.go b/pkg/core/block.go index 94296913..279b8baf 100644 --- a/pkg/core/block.go +++ b/pkg/core/block.go @@ -30,7 +30,7 @@ type Block struct { ParentHash string `db:"parenthash"` Size string `db:"size"` Time int64 `db:"time"` - Transactions []Transaction + Transactions []TransactionModel UncleHash string `db:"uncle_hash"` UnclesReward float64 `db:"uncles_reward"` } diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 2b3af6b6..fb43c655 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -17,6 +17,7 @@ package core import ( + "github.com/ethereum/go-ethereum/common" "math/big" "github.com/ethereum/go-ethereum" @@ -26,10 +27,11 @@ import ( type BlockChain interface { ContractDataFetcher GetBlockByNumber(blockNumber int64) (Block, error) + GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) GetHeaderByNumber(blockNumber int64) (Header, error) GetHeaderByNumbers(blockNumbers []int64) ([]Header, error) GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error) - GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) + GetTransactions(transactionHashes []common.Hash) ([]TransactionModel, error) LastBlock() (*big.Int, error) Node() Node } diff --git a/pkg/core/contract.go b/pkg/core/contract.go index a0da9470..48646dbf 100644 --- a/pkg/core/contract.go +++ b/pkg/core/contract.go @@ -19,5 +19,5 @@ package core type Contract struct { Abi string Hash string - Transactions []Transaction + Transactions []TransactionModel } diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index 6094557c..f9c0515b 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -16,8 +16,8 @@ package core -type Transaction struct { - Data string `db:"input_data"` +type TransactionModel struct { + Data []byte `db:"input_data"` From string `db:"tx_from"` GasLimit uint64 GasPrice int64 @@ -29,3 +29,18 @@ type Transaction struct { TxIndex int64 `db:"tx_index"` Value string } + +type RpcTransaction struct { + Nonce string `json:"nonce"` + GasPrice string `json:"gasPrice"` + GasLimit string `json:"gas"` + Recipient string `json:"to"` + Amount string `json:"value"` + Payload []byte `json:"input"` + V string `json:"v"` + R string `json:"r"` + S string `json:"s"` + Hash string + From string + TransactionIndex string `json:"transactionIndex"` +} diff --git a/pkg/datastore/errors.go b/pkg/datastore/errors.go new file mode 100644 index 00000000..ec734427 --- /dev/null +++ b/pkg/datastore/errors.go @@ -0,0 +1,19 @@ +package datastore + +import "fmt" + +var ErrBlockDoesNotExist = func(blockNumber int64) error { + return fmt.Errorf("Block number %d does not exist", blockNumber) +} + +var ErrContractDoesNotExist = func(contractHash string) error { + return fmt.Errorf("Contract %v does not exist", contractHash) +} + +var ErrFilterDoesNotExist = func(name string) error { + return fmt.Errorf("filter %s does not exist", name) +} + +var ErrReceiptDoesNotExist = func(txHash string) error { + return fmt.Errorf("Receipt for tx: %v does not exist", txHash) +} diff --git a/pkg/datastore/postgres/postgres_test.go b/pkg/datastore/postgres/postgres_test.go index 50dd8b6d..1bf0e92e 100644 --- a/pkg/datastore/postgres/postgres_test.go +++ b/pkg/datastore/postgres/postgres_test.go @@ -88,7 +88,7 @@ var _ = Describe("Postgres DB", func() { badBlock := core.Block{ Number: 123, Nonce: badNonce, - Transactions: []core.Transaction{}, + Transactions: []core.TransactionModel{}, } node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"} db := test_config.NewTestDB(node) @@ -146,10 +146,10 @@ var _ = Describe("Postgres DB", func() { It("does not commit block or transactions if transaction is invalid", func() { //badHash violates db To field length badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) - badTransaction := core.Transaction{To: badHash} + badTransaction := core.TransactionModel{To: badHash} block := core.Block{ Number: 123, - Transactions: []core.Transaction{badTransaction}, + Transactions: []core.TransactionModel{badTransaction}, } node := core.Node{GenesisBlock: "GENESIS", NetworkID: 1, ID: "x123", ClientName: "geth"} db, _ := postgres.NewDB(test_config.DBConfig, node) diff --git a/pkg/datastore/postgres/repositories/block_repository.go b/pkg/datastore/postgres/repositories/block_repository.go index 56d72f33..cdbed4a9 100644 --- a/pkg/datastore/postgres/repositories/block_repository.go +++ b/pkg/datastore/postgres/repositories/block_repository.go @@ -160,7 +160,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err return blockId, nil } -func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.Transaction) error { +func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.TransactionModel) error { for _, transaction := range transactions { err := blockRepository.createTransaction(tx, blockId, transaction) if err != nil { @@ -180,7 +180,7 @@ func nullStringToZero(s string) string { return s } -func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error { +func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.TransactionModel) error { _, err := tx.Exec( `INSERT INTO full_sync_transactions (block_id, gaslimit, gasprice, hash, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") @@ -205,11 +205,11 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in return nil } -func hasLogs(transaction core.Transaction) bool { +func hasLogs(transaction core.TransactionModel) bool { return len(transaction.Receipt.Logs) > 0 } -func hasReceipt(transaction core.Transaction) bool { +func hasReceipt(transaction core.TransactionModel) bool { return transaction.Receipt.TxHash != "" } @@ -302,10 +302,10 @@ func (blockRepository BlockRepository) loadBlock(blockRows *sqlx.Row) (core.Bloc return block.Block, nil } -func (blockRepository BlockRepository) LoadTransactions(transactionRows *sqlx.Rows) []core.Transaction { - var transactions []core.Transaction +func (blockRepository BlockRepository) LoadTransactions(transactionRows *sqlx.Rows) []core.TransactionModel { + var transactions []core.TransactionModel for transactionRows.Next() { - var transaction core.Transaction + var transaction core.TransactionModel err := transactionRows.StructScan(&transaction) if err != nil { log.Fatal(err) diff --git a/pkg/datastore/postgres/repositories/block_repository_test.go b/pkg/datastore/postgres/repositories/block_repository_test.go index 14044209..64b40fcb 100644 --- a/pkg/datastore/postgres/repositories/block_repository_test.go +++ b/pkg/datastore/postgres/repositories/block_repository_test.go @@ -133,7 +133,7 @@ var _ = Describe("Saving blocks", func() { It("saves one transaction associated to the block", func() { block := core.Block{ Number: 123, - Transactions: []core.Transaction{fakes.FakeTransaction}, + Transactions: []core.TransactionModel{fakes.FakeTransaction}, } _, insertErr := blockRepository.CreateOrUpdateBlock(block) @@ -147,7 +147,7 @@ var _ = Describe("Saving blocks", func() { It("saves two transactions associated to the block", func() { block := core.Block{ Number: 123, - Transactions: []core.Transaction{fakes.FakeTransaction, fakes.FakeTransaction}, + Transactions: []core.TransactionModel{fakes.FakeTransaction, fakes.FakeTransaction}, } _, insertErr := blockRepository.CreateOrUpdateBlock(block) @@ -163,7 +163,7 @@ var _ = Describe("Saving blocks", func() { blockOne := core.Block{ Number: 123, Hash: "xabc", - Transactions: []core.Transaction{ + Transactions: []core.TransactionModel{ fakes.GetFakeTransaction("x123", core.Receipt{}), fakes.GetFakeTransaction("x345", core.Receipt{}), }, @@ -171,7 +171,7 @@ var _ = Describe("Saving blocks", func() { blockTwo := core.Block{ Number: 123, Hash: "xdef", - Transactions: []core.Transaction{ + Transactions: []core.TransactionModel{ fakes.GetFakeTransaction("x678", core.Receipt{}), fakes.GetFakeTransaction("x9ab", core.Receipt{}), }, @@ -192,14 +192,14 @@ var _ = Describe("Saving blocks", func() { but block number + node id is`, func() { blockOne := core.Block{ Number: 123, - Transactions: []core.Transaction{ + Transactions: []core.TransactionModel{ fakes.GetFakeTransaction("x123", core.Receipt{}), fakes.GetFakeTransaction("x345", core.Receipt{}), }, } blockTwo := core.Block{ Number: 123, - Transactions: []core.Transaction{ + Transactions: []core.TransactionModel{ fakes.GetFakeTransaction("x678", core.Receipt{}), fakes.GetFakeTransaction("x9ab", core.Receipt{}), }, @@ -256,8 +256,8 @@ var _ = Describe("Saving blocks", func() { var raw bytes.Buffer rlpErr := gethTransaction.EncodeRLP(&raw) Expect(rlpErr).NotTo(HaveOccurred()) - transaction := core.Transaction{ - Data: inputData, + transaction := core.TransactionModel{ + Data: common.Hex2Bytes(inputData), From: from, GasLimit: gasLimit, GasPrice: gasPrice, @@ -271,7 +271,7 @@ var _ = Describe("Saving blocks", func() { } block := core.Block{ Number: 123, - Transactions: []core.Transaction{transaction}, + Transactions: []core.TransactionModel{transaction}, } _, insertErr := blockRepository.CreateOrUpdateBlock(block) diff --git a/pkg/datastore/postgres/repositories/contract_repository_test.go b/pkg/datastore/postgres/repositories/contract_repository_test.go index f2558727..4afdab20 100644 --- a/pkg/datastore/postgres/repositories/contract_repository_test.go +++ b/pkg/datastore/postgres/repositories/contract_repository_test.go @@ -73,24 +73,26 @@ var _ = Describe("Creating contracts", func() { blockRepository = repositories.NewBlockRepository(db) block := core.Block{ Number: 123, - Transactions: []core.Transaction{ + Transactions: []core.TransactionModel{ {Hash: "TRANSACTION1", To: "x123", Value: "0"}, {Hash: "TRANSACTION2", To: "x345", Value: "0"}, {Hash: "TRANSACTION3", To: "x123", Value: "0"}, }, } - blockRepository.CreateOrUpdateBlock(block) + _, insertBlockErr := blockRepository.CreateOrUpdateBlock(block) + Expect(insertBlockErr).NotTo(HaveOccurred()) - contractRepository.CreateContract(core.Contract{Hash: "x123"}) + insertContractErr := contractRepository.CreateContract(core.Contract{Hash: "x123"}) + Expect(insertContractErr).NotTo(HaveOccurred()) contract, err := contractRepository.GetContract("x123") Expect(err).ToNot(HaveOccurred()) sort.Slice(contract.Transactions, func(i, j int) bool { return contract.Transactions[i].Hash < contract.Transactions[j].Hash }) Expect(contract.Transactions).To( - Equal([]core.Transaction{ - {Hash: "TRANSACTION1", To: "x123", Value: "0"}, - {Hash: "TRANSACTION3", To: "x123", Value: "0"}, + Equal([]core.TransactionModel{ + {Data: []byte{}, Hash: "TRANSACTION1", To: "x123", Value: "0"}, + {Data: []byte{}, Hash: "TRANSACTION3", To: "x123", Value: "0"}, })) }) diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 1f886003..f2235365 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -49,11 +49,13 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int return 0, ErrValidHeaderExists } -func (repository HeaderRepository) CreateTransaction(headerID int64, transaction core.Transaction) error { +func (repository HeaderRepository) CreateTransaction(headerID int64, transaction core.TransactionModel) error { _, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions - (header_id, hash, tx_to, tx_from, tx_index) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT DO NOTHING`, - headerID, transaction.Hash, transaction.To, transaction.From, transaction.TxIndex) + (header_id, hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") + VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC) + ON CONFLICT DO NOTHING`, headerID, transaction.Hash, transaction.GasLimit, transaction.GasPrice, + transaction.Data, transaction.Nonce, transaction.Raw, transaction.From, transaction.TxIndex, transaction.To, + transaction.Value) return err } diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index d2dbd33c..ef491306 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -188,18 +188,26 @@ var _ = Describe("Block header repository", func() { toAddress := common.HexToAddress("0x5678") txHash := common.HexToHash("0x9876") txIndex := big.NewInt(123) - transaction := core.Transaction{ - From: fromAddress.Hex(), - Hash: txHash.Hex(), - To: toAddress.Hex(), - TxIndex: txIndex.Int64(), + transaction := core.TransactionModel{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", } insertErr := repo.CreateTransaction(headerID, transaction) Expect(insertErr).NotTo(HaveOccurred()) - var dbTransaction core.Transaction - err = db.Get(&dbTransaction, `SELECT hash, tx_from, tx_to, tx_index FROM public.light_sync_transactions WHERE header_id = $1`, headerID) + var dbTransaction core.TransactionModel + err = db.Get(&dbTransaction, + `SELECT hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value" + FROM public.light_sync_transactions WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(dbTransaction).To(Equal(transaction)) }) @@ -211,11 +219,18 @@ var _ = Describe("Block header repository", func() { toAddress := common.HexToAddress("0x5678") txHash := common.HexToHash("0x9876") txIndex := big.NewInt(123) - transaction := core.Transaction{ - From: fromAddress.Hex(), - Hash: txHash.Hex(), - To: toAddress.Hex(), - TxIndex: txIndex.Int64(), + transaction := core.TransactionModel{ + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 0, + GasPrice: 0, + Hash: txHash.Hex(), + Nonce: 0, + Raw: []byte{}, + Receipt: core.Receipt{}, + To: toAddress.Hex(), + TxIndex: txIndex.Int64(), + Value: "0", } insertErr := repo.CreateTransaction(headerID, transaction) @@ -224,8 +239,10 @@ var _ = Describe("Block header repository", func() { insertTwoErr := repo.CreateTransaction(headerID, transaction) Expect(insertTwoErr).NotTo(HaveOccurred()) - var dbTransactions []core.Transaction - err = db.Select(&dbTransactions, `SELECT hash, tx_from, tx_to, tx_index FROM public.light_sync_transactions WHERE header_id = $1`, headerID) + var dbTransactions []core.TransactionModel + err = db.Select(&dbTransactions, + `SELECT hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value" + FROM public.light_sync_transactions WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(len(dbTransactions)).To(Equal(1)) }) diff --git a/pkg/datastore/postgres/repositories/logs_repository_test.go b/pkg/datastore/postgres/repositories/logs_repository_test.go index 7cf319a4..fe5d069d 100644 --- a/pkg/datastore/postgres/repositories/logs_repository_test.go +++ b/pkg/datastore/postgres/repositories/logs_repository_test.go @@ -204,7 +204,7 @@ var _ = Describe("Logs Repository", func() { } transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt) - block := core.Block{Transactions: []core.Transaction{transaction}} + block := core.Block{Transactions: []core.TransactionModel{transaction}} _, err := blockRepository.CreateOrUpdateBlock(block) Expect(err).To(Not(HaveOccurred())) retrievedLogs, err := logsRepository.GetLogs("0x99041f808d598b782d5a3e498681c2452a31da08", 4745407) diff --git a/pkg/datastore/postgres/repositories/receipts_repository_test.go b/pkg/datastore/postgres/repositories/receipt_repository_test.go similarity index 96% rename from pkg/datastore/postgres/repositories/receipts_repository_test.go rename to pkg/datastore/postgres/repositories/receipt_repository_test.go index b2e4433a..76d1516b 100644 --- a/pkg/datastore/postgres/repositories/receipts_repository_test.go +++ b/pkg/datastore/postgres/repositories/receipt_repository_test.go @@ -27,7 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("Receipts Repository", func() { +var _ = Describe("Receipt Repository", func() { var blockRepository datastore.BlockRepository var logRepository datastore.LogRepository var receiptRepository datastore.ReceiptRepository @@ -119,7 +119,7 @@ var _ = Describe("Receipts Repository", func() { } transaction := fakes.GetFakeTransaction(expected.TxHash, expected) - block := core.Block{Transactions: []core.Transaction{transaction}} + block := core.Block{Transactions: []core.TransactionModel{transaction}} _, err := blockRepository.CreateOrUpdateBlock(block) @@ -148,7 +148,7 @@ var _ = Describe("Receipts Repository", func() { transaction := fakes.GetFakeTransaction(receipt.TxHash, receipt) block := core.Block{ - Transactions: []core.Transaction{transaction}, + Transactions: []core.TransactionModel{transaction}, } _, err := blockRepository.CreateOrUpdateBlock(block) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index b26bb8ce..0b5af3dd 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -17,16 +17,10 @@ package datastore import ( - "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" ) -var ErrBlockDoesNotExist = func(blockNumber int64) error { - return fmt.Errorf("Block number %d does not exist", blockNumber) -} - type BlockRepository interface { CreateOrUpdateBlock(block core.Block) (int64, error) GetBlock(blockNumber int64) (core.Block, error) @@ -34,20 +28,12 @@ type BlockRepository interface { SetBlocksStatus(chainHead int64) error } -var ErrContractDoesNotExist = func(contractHash string) error { - return fmt.Errorf("Contract %v does not exist", contractHash) -} - type ContractRepository interface { CreateContract(contract core.Contract) error GetContract(contractHash string) (core.Contract, error) ContractExists(contractHash string) (bool, error) } -var ErrFilterDoesNotExist = func(name string) error { - return fmt.Errorf("filter %s does not exist", name) -} - type FilterRepository interface { CreateFilter(filter filters.LogFilter) error GetFilter(name string) (filters.LogFilter, error) @@ -55,7 +41,7 @@ type FilterRepository interface { type HeaderRepository interface { CreateOrUpdateHeader(header core.Header) (int64, error) - CreateTransaction(headerID int64, transaction core.Transaction) error + CreateTransaction(headerID int64, transaction core.TransactionModel) error GetHeader(blockNumber int64) (core.Header, error) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) } @@ -65,10 +51,6 @@ type LogRepository interface { GetLogs(address string, blockNumber int64) ([]core.Log, error) } -var ErrReceiptDoesNotExist = func(txHash string) error { - return fmt.Errorf("Receipt for tx: %v does not exist", txHash) -} - type ReceiptRepository interface { CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) diff --git a/pkg/fakes/data.go b/pkg/fakes/data.go index eeb25fe9..f8549a0a 100644 --- a/pkg/fakes/data.go +++ b/pkg/fakes/data.go @@ -29,6 +29,7 @@ import ( ) var ( + FakeAddress = common.HexToAddress("0x1234567890abcdef") FakeError = errors.New("failed") FakeHash = common.BytesToHash([]byte{1, 2, 3, 4, 5}) fakeTimestamp = int64(111111111) @@ -53,8 +54,8 @@ func GetFakeHeader(blockNumber int64) core.Header { var fakeTransaction types.Transaction var rawTransaction bytes.Buffer var _ = fakeTransaction.EncodeRLP(&rawTransaction) -var FakeTransaction = core.Transaction{ - Data: "", +var FakeTransaction = core.TransactionModel{ + Data: []byte{}, From: "", GasLimit: 0, GasPrice: 0, @@ -67,15 +68,15 @@ var FakeTransaction = core.Transaction{ Value: "0", } -func GetFakeTransaction(hash string, receipt core.Receipt) core.Transaction { +func GetFakeTransaction(hash string, receipt core.Receipt) core.TransactionModel { gethTransaction := types.Transaction{} var raw bytes.Buffer err := gethTransaction.EncodeRLP(&raw) if err != nil { panic("failed to marshal transaction creating test fake") } - return core.Transaction{ - Data: "", + return core.TransactionModel{ + Data: []byte{}, From: "", GasLimit: 0, GasPrice: 0, diff --git a/pkg/fakes/mock_blockchain.go b/pkg/fakes/mock_blockchain.go index 67fab6d0..1f337781 100644 --- a/pkg/fakes/mock_blockchain.go +++ b/pkg/fakes/mock_blockchain.go @@ -17,6 +17,7 @@ package fakes import ( + "github.com/ethereum/go-ethereum/common" "math/big" "github.com/ethereum/go-ethereum" @@ -35,11 +36,15 @@ type MockBlockChain struct { fetchContractDataPassedResult interface{} fetchContractDataPassedBlockNumber int64 getBlockByNumberErr error + GetTransactionsCalled bool + GetTransactionsError error + GetTransactionsPassedHashes []common.Hash logQuery ethereum.FilterQuery logQueryErr error logQueryReturnLogs []types.Log lastBlock *big.Int node core.Node + Transactions []core.TransactionModel } func NewMockBlockChain() *MockBlockChain { @@ -104,6 +109,12 @@ func (chain *MockBlockChain) GetLogs(contract core.Contract, startingBlockNumber return []core.Log{}, nil } +func (chain *MockBlockChain) GetTransactions(transactionHashes []common.Hash) ([]core.TransactionModel, error) { + chain.GetTransactionsCalled = true + chain.GetTransactionsPassedHashes = transactionHashes + return chain.Transactions, chain.GetTransactionsError +} + func (chain *MockBlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { return []byte{}, nil } diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index ef196949..c89d8f16 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -27,6 +27,8 @@ type MockHeaderRepository struct { createOrUpdateHeaderErr error createOrUpdateHeaderPassedBlockNumbers []int64 createOrUpdateHeaderReturnID int64 + CreateTransactionCalled bool + CreateTransactionError error getHeaderError error getHeaderReturnBlockHash string missingBlockNumbers []int64 @@ -56,8 +58,9 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr } -func (repository *MockHeaderRepository) CreateTransaction(headerID int64, transaction core.Transaction) error { - panic("implement me") +func (repository *MockHeaderRepository) CreateTransaction(headerID int64, transaction core.TransactionModel) error { + repository.CreateTransactionCalled = true + return repository.CreateTransactionError } func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { diff --git a/pkg/fakes/mock_transaction_converter.go b/pkg/fakes/mock_transaction_converter.go index dd535acd..5f23310d 100644 --- a/pkg/fakes/mock_transaction_converter.go +++ b/pkg/fakes/mock_transaction_converter.go @@ -18,39 +18,30 @@ package fakes import ( "github.com/ethereum/go-ethereum/core/types" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" ) type MockTransactionConverter struct { - convertTransactionsToCoreCalled bool - convertTransactionsToCorePassedBlock *types.Block - convertTransactionsToCoreReturnTransactions []core.Transaction - convertTransactionsToCoreReturnError error + ConvertHeaderTransactionIndexToIntCalled bool + ConvertBlockTransactionsToCoreCalled bool + ConvertBlockTransactionsToCorePassedBlock *types.Block } func NewMockTransactionConverter() *MockTransactionConverter { return &MockTransactionConverter{ - convertTransactionsToCoreCalled: false, - convertTransactionsToCorePassedBlock: nil, - convertTransactionsToCoreReturnTransactions: nil, - convertTransactionsToCoreReturnError: nil, + ConvertHeaderTransactionIndexToIntCalled: false, + ConvertBlockTransactionsToCoreCalled: false, + ConvertBlockTransactionsToCorePassedBlock: nil, } } -func (mtc *MockTransactionConverter) SetConvertTransactionsToCoreReturnVals(transactions []core.Transaction, err error) { - mtc.convertTransactionsToCoreReturnTransactions = transactions - mtc.convertTransactionsToCoreReturnError = err +func (converter *MockTransactionConverter) ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error) { + converter.ConvertBlockTransactionsToCoreCalled = true + converter.ConvertBlockTransactionsToCorePassedBlock = gethBlock + return []core.TransactionModel{}, nil } -func (mtc *MockTransactionConverter) ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) { - mtc.convertTransactionsToCoreCalled = true - mtc.convertTransactionsToCorePassedBlock = gethBlock - return mtc.convertTransactionsToCoreReturnTransactions, mtc.convertTransactionsToCoreReturnError -} - -func (mtc *MockTransactionConverter) AssertConvertTransactionsToCoreCalledWith(gethBlock *types.Block) { - Expect(mtc.convertTransactionsToCoreCalled).To(BeTrue()) - Expect(mtc.convertTransactionsToCorePassedBlock).To(Equal(gethBlock)) +func (converter *MockTransactionConverter) ConvertRpcTransactionsToModels(transactions []core.RpcTransaction) ([]core.TransactionModel, error) { + converter.ConvertHeaderTransactionIndexToIntCalled = true + return nil, nil } diff --git a/pkg/fakes/mock_transaction_syncer.go b/pkg/fakes/mock_transaction_syncer.go new file mode 100644 index 00000000..0753a69a --- /dev/null +++ b/pkg/fakes/mock_transaction_syncer.go @@ -0,0 +1,13 @@ +package fakes + +import "github.com/ethereum/go-ethereum/core/types" + +type MockTransactionSyncer struct { + SyncTransactionsCalled bool + SyncTransactionsError error +} + +func (syncer *MockTransactionSyncer) SyncTransactions(headerID int64, logs []types.Log) error { + syncer.SyncTransactionsCalled = true + return syncer.SyncTransactionsError +} diff --git a/pkg/geth/blockchain.go b/pkg/geth/blockchain.go index ea2a653a..6a16a917 100644 --- a/pkg/geth/blockchain.go +++ b/pkg/geth/blockchain.go @@ -18,10 +18,11 @@ package geth import ( "errors" + "fmt" + "github.com/ethereum/go-ethereum" "math/big" "strconv" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -37,20 +38,22 @@ var ErrEmptyHeader = errors.New("empty header returned over RPC") const MAX_BATCH_SIZE = 100 type BlockChain struct { - blockConverter vulcCommon.BlockConverter - ethClient core.EthClient - headerConverter vulcCommon.HeaderConverter - node core.Node - rpcClient core.RpcClient + blockConverter vulcCommon.BlockConverter + ethClient core.EthClient + headerConverter vulcCommon.HeaderConverter + node core.Node + rpcClient core.RpcClient + transactionConverter vulcCommon.TransactionConverter } func NewBlockChain(ethClient core.EthClient, rpcClient core.RpcClient, node core.Node, converter vulcCommon.TransactionConverter) *BlockChain { return &BlockChain{ - blockConverter: vulcCommon.NewBlockConverter(converter), - ethClient: ethClient, - headerConverter: vulcCommon.HeaderConverter{}, - node: node, - rpcClient: rpcClient, + blockConverter: vulcCommon.NewBlockConverter(converter), + ethClient: ethClient, + headerConverter: vulcCommon.HeaderConverter{}, + node: node, + rpcClient: rpcClient, + transactionConverter: converter, } } @@ -62,6 +65,14 @@ func (blockChain *BlockChain) GetBlockByNumber(blockNumber int64) (block core.Bl return blockChain.blockConverter.ToCoreBlock(gethBlock) } +func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) { + gethLogs, err := blockChain.ethClient.FilterLogs(context.Background(), query) + if err != nil { + return []types.Log{}, err + } + return gethLogs, nil +} + func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.Header, err error) { if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID { return blockChain.getPOAHeader(blockNumber) @@ -76,49 +87,55 @@ func (blockChain *BlockChain) GetHeaderByNumbers(blockNumbers []int64) (header [ return blockChain.getPOWHeaders(blockNumbers) } -func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Header, err error) { - gethHeader, err := blockChain.ethClient.HeaderByNumber(context.Background(), big.NewInt(blockNumber)) - if err != nil { - return header, err +func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) { + if endingBlockNumber == nil { + endingBlockNumber = startingBlockNumber } - return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String()), nil + contractAddress := common.HexToAddress(contract.Hash) + fc := ethereum.FilterQuery{ + FromBlock: startingBlockNumber, + ToBlock: endingBlockNumber, + Addresses: []common.Address{contractAddress}, + Topics: nil, + } + gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc) + if err != nil { + return []core.Log{}, err + } + logs := vulcCommon.ToCoreLogs(gethLogs) + return logs, nil } -func (blockChain *BlockChain) getPOWHeaders(blockNumbers []int64) (headers []core.Header, err error) { +func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) ([]core.TransactionModel, error) { + numTransactions := len(transactionHashes) var batch []client.BatchElem - var POWHeaders [MAX_BATCH_SIZE]types.Header - includeTransactions := false - - for index, blockNumber := range blockNumbers { - - if index >= MAX_BATCH_SIZE { - break - } - - blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber)) + transactions := make([]core.RpcTransaction, numTransactions) + for index, transactionHash := range transactionHashes { batchElem := client.BatchElem{ - Method: "eth_getBlockByNumber", - Result: &POWHeaders[index], - Args: []interface{}{blockNumberArg, includeTransactions}, + Method: "eth_getTransactionByHash", + Result: &transactions[index], + Args: []interface{}{transactionHash}, } - batch = append(batch, batchElem) } - err = blockChain.rpcClient.BatchCall(batch) - if err != nil { - return headers, err + rpcErr := blockChain.rpcClient.BatchCall(batch) + if rpcErr != nil { + fmt.Println("rpc err") + return []core.TransactionModel{}, rpcErr } - for _, POWHeader := range POWHeaders { - if POWHeader.Number != nil { - header := blockChain.headerConverter.Convert(&POWHeader, POWHeader.Hash().String()) - headers = append(headers, header) - } - } + return blockChain.transactionConverter.ConvertRpcTransactionsToModels(transactions) +} - return headers, err +func (blockChain *BlockChain) LastBlock() (*big.Int, error) { + block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil) + return block.Number, err +} + +func (blockChain *BlockChain) Node() core.Node { + return blockChain.node } func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Header, err error) { @@ -204,38 +221,47 @@ func (blockChain *BlockChain) getPOAHeaders(blockNumbers []int64) (headers []cor return headers, err } -func (blockChain *BlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) { - if endingBlockNumber == nil { - endingBlockNumber = startingBlockNumber - } - contractAddress := common.HexToAddress(contract.Hash) - fc := ethereum.FilterQuery{ - FromBlock: startingBlockNumber, - ToBlock: endingBlockNumber, - Addresses: []common.Address{contractAddress}, - Topics: nil, - } - gethLogs, err := blockChain.GetEthLogsWithCustomQuery(fc) +func (blockChain *BlockChain) getPOWHeader(blockNumber int64) (header core.Header, err error) { + gethHeader, err := blockChain.ethClient.HeaderByNumber(context.Background(), big.NewInt(blockNumber)) if err != nil { - return []core.Log{}, err + return header, err } - logs := vulcCommon.ToCoreLogs(gethLogs) - return logs, nil + return blockChain.headerConverter.Convert(gethHeader, gethHeader.Hash().String()), nil } -func (blockChain *BlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) { - gethLogs, err := blockChain.ethClient.FilterLogs(context.Background(), query) +func (blockChain *BlockChain) getPOWHeaders(blockNumbers []int64) (headers []core.Header, err error) { + var batch []client.BatchElem + var POWHeaders [MAX_BATCH_SIZE]types.Header + includeTransactions := false + + for index, blockNumber := range blockNumbers { + + if index >= MAX_BATCH_SIZE { + break + } + + blockNumberArg := hexutil.EncodeBig(big.NewInt(blockNumber)) + + batchElem := client.BatchElem{ + Method: "eth_getBlockByNumber", + Result: &POWHeaders[index], + Args: []interface{}{blockNumberArg, includeTransactions}, + } + + batch = append(batch, batchElem) + } + + err = blockChain.rpcClient.BatchCall(batch) if err != nil { - return []types.Log{}, err + return headers, err } - return gethLogs, nil -} -func (blockChain *BlockChain) LastBlock() (*big.Int, error) { - block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil) - return block.Number, err -} + for _, POWHeader := range POWHeaders { + if POWHeader.Number != nil { + header := blockChain.headerConverter.Convert(&POWHeader, POWHeader.Hash().String()) + headers = append(headers, header) + } + } -func (blockChain *BlockChain) Node() core.Node { - return blockChain.node + return headers, err } diff --git a/pkg/geth/blockchain_test.go b/pkg/geth/blockchain_test.go index 27667fe8..292e2cd5 100644 --- a/pkg/geth/blockchain_test.go +++ b/pkg/geth/blockchain_test.go @@ -30,20 +30,23 @@ import ( vulcCore "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db" ) var _ = Describe("Geth blockchain", func() { - var mockClient *fakes.MockEthClient - var mockRpcClient *fakes.MockRpcClient - var node vulcCore.Node - var blockChain *geth.BlockChain + var ( + mockClient *fakes.MockEthClient + blockChain *geth.BlockChain + mockRpcClient *fakes.MockRpcClient + mockTransactionConverter *fakes.MockTransactionConverter + node vulcCore.Node + ) BeforeEach(func() { mockClient = fakes.NewMockEthClient() mockRpcClient = fakes.NewMockRpcClient() + mockTransactionConverter = fakes.NewMockTransactionConverter() node = vulcCore.Node{} - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) + blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, mockTransactionConverter) }) Describe("getting a block", func() { @@ -89,8 +92,6 @@ var _ = Describe("Geth blockchain", func() { }) It("fetches headers with multiple blocks", func() { - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) - _, err := blockChain.GetHeaderByNumbers([]int64{100, 99}) Expect(err).NotTo(HaveOccurred()) @@ -103,7 +104,7 @@ var _ = Describe("Geth blockchain", func() { node.NetworkID = vulcCore.KOVAN_NETWORK_ID blockNumber := hexutil.Big(*big.NewInt(100)) mockRpcClient.SetReturnPOAHeader(vulcCore.POAHeader{Number: &blockNumber}) - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) + blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter()) _, err := blockChain.GetHeaderByNumber(100) @@ -114,7 +115,7 @@ var _ = Describe("Geth blockchain", func() { It("returns err if rpcClient returns err", func() { node.NetworkID = vulcCore.KOVAN_NETWORK_ID mockRpcClient.SetCallContextErr(fakes.FakeError) - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) + blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter()) _, err := blockChain.GetHeaderByNumber(100) @@ -124,7 +125,7 @@ var _ = Describe("Geth blockchain", func() { It("returns error if returned header is empty", func() { node.NetworkID = vulcCore.KOVAN_NETWORK_ID - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) + blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, fakes.NewMockTransactionConverter()) _, err := blockChain.GetHeaderByNumber(100) @@ -136,7 +137,6 @@ var _ = Describe("Geth blockchain", func() { node.NetworkID = vulcCore.KOVAN_NETWORK_ID blockNumber := hexutil.Big(*big.NewInt(100)) mockRpcClient.SetReturnPOAHeaders([]vulcCore.POAHeader{{Number: &blockNumber}}) - blockChain = geth.NewBlockChain(mockClient, mockRpcClient, node, cold_db.NewColdDbTransactionConverter()) _, err := blockChain.GetHeaderByNumbers([]int64{100, 99}) @@ -216,6 +216,22 @@ var _ = Describe("Geth blockchain", func() { }) }) + Describe("getting transactions", func() { + It("fetches transaction for each hash", func() { + _, err := blockChain.GetTransactions([]common.Hash{{}, {}}) + + Expect(err).NotTo(HaveOccurred()) + mockRpcClient.AssertBatchCalledWith("eth_getTransactionByHash", 2) + }) + + It("converts transaction indexes from hex to int", func() { + _, err := blockChain.GetTransactions([]common.Hash{{}, {}}) + + Expect(err).NotTo(HaveOccurred()) + Expect(mockTransactionConverter.ConvertHeaderTransactionIndexToIntCalled).To(BeTrue()) + }) + }) + Describe("getting the most recent block number", func() { It("fetches latest header from ethClient", func() { blockNumber := int64(100) diff --git a/pkg/geth/cold_import/importer_test.go b/pkg/geth/cold_import/importer_test.go index 26e5a8aa..58a5145e 100644 --- a/pkg/geth/cold_import/importer_test.go +++ b/pkg/geth/cold_import/importer_test.go @@ -81,7 +81,8 @@ var _ = Describe("Geth cold importer", func() { mockEthereumDatabase.AssertGetBlockHashCalledWith(blockNumber) mockEthereumDatabase.AssertGetBlockCalledWith(fakeHash, blockNumber) - mockTransactionConverter.AssertConvertTransactionsToCoreCalledWith(fakeGethBlock) + Expect(mockTransactionConverter.ConvertBlockTransactionsToCoreCalled).To(BeTrue()) + Expect(mockTransactionConverter.ConvertBlockTransactionsToCorePassedBlock).To(Equal(fakeGethBlock)) convertedBlock, err := blockConverter.ToCoreBlock(fakeGethBlock) Expect(err).NotTo(HaveOccurred()) mockBlockRepository.AssertCreateOrUpdateBlockCalledWith(convertedBlock) diff --git a/pkg/geth/converters/cold_db/transaction_converter.go b/pkg/geth/converters/cold_db/transaction_converter.go index 0bd409d1..fd6a319e 100644 --- a/pkg/geth/converters/cold_db/transaction_converter.go +++ b/pkg/geth/converters/cold_db/transaction_converter.go @@ -18,7 +18,6 @@ package cold_db import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" "golang.org/x/sync/errgroup" @@ -31,9 +30,9 @@ func NewColdDbTransactionConverter() *ColdDbTransactionConverter { return &ColdDbTransactionConverter{} } -func (cdtc *ColdDbTransactionConverter) ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) { +func (cdtc *ColdDbTransactionConverter) ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error) { var g errgroup.Group - coreTransactions := make([]core.Transaction, len(gethBlock.Transactions())) + coreTransactions := make([]core.TransactionModel, len(gethBlock.Transactions())) for gethTransactionIndex, gethTransaction := range gethBlock.Transactions() { transaction := gethTransaction @@ -55,6 +54,10 @@ func (cdtc *ColdDbTransactionConverter) ConvertTransactionsToCore(gethBlock *typ return coreTransactions, nil } +func (cdtc *ColdDbTransactionConverter) ConvertRpcTransactionsToModels(transactions []core.RpcTransaction) ([]core.TransactionModel, error) { + panic("converting transaction indexes to integer not supported for cold import") +} + func getSigner(tx *types.Transaction) types.Signer { v, _, _ := tx.RawSignatureValues() if v.Sign() != 0 && tx.Protected() { @@ -63,9 +66,8 @@ func getSigner(tx *types.Transaction) types.Signer { return types.HomesteadSigner{} } -func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.Transaction { - data := hexutil.Encode(transaction.Data()) - return core.Transaction{ +func transToCoreTrans(transaction *types.Transaction, from *common.Address) core.TransactionModel { + return core.TransactionModel{ Hash: transaction.Hash().Hex(), Nonce: transaction.Nonce(), To: strings.ToLower(addressToHex(transaction.To())), @@ -73,7 +75,7 @@ func transToCoreTrans(transaction *types.Transaction, from *common.Address) core GasLimit: transaction.Gas(), GasPrice: transaction.GasPrice().Int64(), Value: transaction.Value().String(), - Data: data, + Data: transaction.Data(), } } diff --git a/pkg/geth/converters/common/block_converter.go b/pkg/geth/converters/common/block_converter.go index 52e1d245..c12d095b 100644 --- a/pkg/geth/converters/common/block_converter.go +++ b/pkg/geth/converters/common/block_converter.go @@ -32,7 +32,7 @@ func NewBlockConverter(transactionConverter TransactionConverter) BlockConverter } func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error) { - transactions, err := bc.transactionConverter.ConvertTransactionsToCore(gethBlock) + transactions, err := bc.transactionConverter.ConvertBlockTransactionsToCore(gethBlock) if err != nil { return core.Block{}, err } diff --git a/pkg/geth/converters/common/block_converter_test.go b/pkg/geth/converters/common/block_converter_test.go index cf4ac8d8..c98d7d3e 100644 --- a/pkg/geth/converters/common/block_converter_test.go +++ b/pkg/geth/converters/common/block_converter_test.go @@ -260,7 +260,8 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(coreBlock.Transactions)).To(Equal(1)) coreTransaction := coreBlock.Transactions[0] - Expect(coreTransaction.Data).To(Equal("0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14")) + expectedData := common.FromHex("0xf7d8c8830000000000000000000000000000000000000000000000000000000000037788000000000000000000000000000000000000000000000000000000000003bd14") + Expect(coreTransaction.Data).To(Equal(expectedData)) Expect(coreTransaction.To).To(Equal(gethTransaction.To().Hex())) Expect(coreTransaction.From).To(Equal("0x0000000000000000000000000000000000000123")) Expect(coreTransaction.GasLimit).To(Equal(gethTransaction.Gas())) diff --git a/pkg/geth/converters/common/transaction_converter.go b/pkg/geth/converters/common/transaction_converter.go index 885b7488..fb9584f6 100644 --- a/pkg/geth/converters/common/transaction_converter.go +++ b/pkg/geth/converters/common/transaction_converter.go @@ -22,5 +22,6 @@ import ( ) type TransactionConverter interface { - ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) + ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error) + ConvertRpcTransactionsToModels(transactions []core.RpcTransaction) ([]core.TransactionModel, error) } diff --git a/pkg/geth/converters/rpc/rpc_suite_test.go b/pkg/geth/converters/rpc/rpc_suite_test.go new file mode 100644 index 00000000..346968fa --- /dev/null +++ b/pkg/geth/converters/rpc/rpc_suite_test.go @@ -0,0 +1,13 @@ +package rpc_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRpc(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Rpc Suite") +} diff --git a/pkg/geth/converters/rpc/transaction_converter.go b/pkg/geth/converters/rpc/transaction_converter.go index 78f6e9c6..067a1e99 100644 --- a/pkg/geth/converters/rpc/transaction_converter.go +++ b/pkg/geth/converters/rpc/transaction_converter.go @@ -19,11 +19,13 @@ package rpc import ( "bytes" "context" + "fmt" "log" + "math/big" "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/core/types" "golang.org/x/sync/errgroup" @@ -35,29 +37,75 @@ type RpcTransactionConverter struct { client core.EthClient } +// raw transaction data, required for generating RLP +type transactionData struct { + AccountNonce uint64 + Price *big.Int + GasLimit uint64 + Recipient *common.Address `rlp:"nil"` // nil means contract creation + Amount *big.Int + Payload []byte + V *big.Int + R *big.Int + S *big.Int +} + func NewRpcTransactionConverter(client core.EthClient) *RpcTransactionConverter { return &RpcTransactionConverter{client: client} } -func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.Block) ([]core.Transaction, error) { +func (converter *RpcTransactionConverter) ConvertRpcTransactionsToModels(transactions []core.RpcTransaction) ([]core.TransactionModel, error) { + var results []core.TransactionModel + for _, transaction := range transactions { + txData, convertErr := getTransactionData(transaction) + if convertErr != nil { + return nil, convertErr + } + txRLP, rlpErr := getTransactionRLP(txData) + if rlpErr != nil { + return nil, rlpErr + } + txIndex, txIndexErr := hexToBigInt(transaction.TransactionIndex) + if txIndexErr != nil { + return nil, txIndexErr + } + transactionModel := core.TransactionModel{ + Data: transaction.Payload, + From: transaction.From, + GasLimit: txData.GasLimit, + GasPrice: txData.Price.Int64(), + Hash: transaction.Hash, + Nonce: txData.AccountNonce, + Raw: txRLP, + // NOTE: Light Sync transactions don't include receipt; would require separate RPC call + To: transaction.Recipient, + TxIndex: txIndex.Int64(), + Value: txData.Amount.String(), + } + results = append(results, transactionModel) + } + return results, nil +} + +func (converter *RpcTransactionConverter) ConvertBlockTransactionsToCore(gethBlock *types.Block) ([]core.TransactionModel, error) { var g errgroup.Group - coreTransactions := make([]core.Transaction, len(gethBlock.Transactions())) + coreTransactions := make([]core.TransactionModel, len(gethBlock.Transactions())) for gethTransactionIndex, gethTransaction := range gethBlock.Transactions() { //https://golang.org/doc/faq#closures_and_goroutines transaction := gethTransaction transactionIndex := uint(gethTransactionIndex) g.Go(func() error { - from, err := rtc.client.TransactionSender(context.Background(), transaction, gethBlock.Hash(), transactionIndex) + from, err := converter.client.TransactionSender(context.Background(), transaction, gethBlock.Hash(), transactionIndex) if err != nil { log.Println("transaction sender: ", err) return err } - coreTransaction, convertErr := transToCoreTrans(transaction, &from, int64(gethTransactionIndex)) + coreTransaction, convertErr := convertGethTransactionToModel(transaction, &from, int64(gethTransactionIndex)) if convertErr != nil { return convertErr } - coreTransaction, err = rtc.appendReceiptToTransaction(coreTransaction) + coreTransaction, err = converter.appendReceiptToTransaction(coreTransaction) if err != nil { log.Println("receipt: ", err) return err @@ -73,7 +121,7 @@ func (rtc *RpcTransactionConverter) ConvertTransactionsToCore(gethBlock *types.B return coreTransactions, nil } -func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core.Transaction) (core.Transaction, error) { +func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core.TransactionModel) (core.TransactionModel, error) { gethReceipt, err := rtc.client.TransactionReceipt(context.Background(), common.HexToHash(transaction.Hash)) if err != nil { return transaction, err @@ -83,15 +131,14 @@ func (rtc *RpcTransactionConverter) appendReceiptToTransaction(transaction core. return transaction, nil } -func transToCoreTrans(transaction *types.Transaction, from *common.Address, transactionIndex int64) (core.Transaction, error) { - data := hexutil.Encode(transaction.Data()) - var raw bytes.Buffer +func convertGethTransactionToModel(transaction *types.Transaction, from *common.Address, transactionIndex int64) (core.TransactionModel, error) { + raw := bytes.Buffer{} encodeErr := transaction.EncodeRLP(&raw) if encodeErr != nil { - return core.Transaction{}, encodeErr + return core.TransactionModel{}, encodeErr } - return core.Transaction{ - Data: data, + return core.TransactionModel{ + Data: transaction.Data(), From: strings.ToLower(addressToHex(from)), GasLimit: transaction.Gas(), GasPrice: transaction.GasPrice().Int64(), @@ -104,9 +151,70 @@ func transToCoreTrans(transaction *types.Transaction, from *common.Address, tran }, nil } +func getTransactionData(transaction core.RpcTransaction) (transactionData, error) { + nonce, nonceErr := hexToBigInt(transaction.Nonce) + if nonceErr != nil { + return transactionData{}, nonceErr + } + gasPrice, gasPriceErr := hexToBigInt(transaction.GasPrice) + if gasPriceErr != nil { + return transactionData{}, gasPriceErr + } + gasLimit, gasLimitErr := hexToBigInt(transaction.GasLimit) + if gasLimitErr != nil { + return transactionData{}, gasLimitErr + } + recipient := common.HexToAddress(transaction.Recipient) + amount, amountErr := hexToBigInt(transaction.Amount) + if amountErr != nil { + return transactionData{}, amountErr + } + v, vErr := hexToBigInt(transaction.V) + if vErr != nil { + return transactionData{}, vErr + } + r, rErr := hexToBigInt(transaction.R) + if rErr != nil { + return transactionData{}, rErr + } + s, sErr := hexToBigInt(transaction.S) + if sErr != nil { + return transactionData{}, sErr + } + return transactionData{ + AccountNonce: nonce.Uint64(), + Price: gasPrice, + GasLimit: gasLimit.Uint64(), + Recipient: &recipient, + Amount: amount, + Payload: transaction.Payload, + V: v, + R: r, + S: s, + }, nil +} + +func getTransactionRLP(txData transactionData) ([]byte, error) { + transactionRlp := bytes.Buffer{} + encodeErr := rlp.Encode(&transactionRlp, txData) + if encodeErr != nil { + return nil, encodeErr + } + return transactionRlp.Bytes(), nil +} + func addressToHex(to *common.Address) string { if to == nil { return "" } return to.Hex() } + +func hexToBigInt(hex string) (*big.Int, error) { + result := big.NewInt(0) + _, scanErr := fmt.Sscan(hex, result) + if scanErr != nil { + return nil, scanErr + } + return result, nil +} diff --git a/pkg/geth/converters/rpc/transaction_converter_test.go b/pkg/geth/converters/rpc/transaction_converter_test.go new file mode 100644 index 00000000..705e0572 --- /dev/null +++ b/pkg/geth/converters/rpc/transaction_converter_test.go @@ -0,0 +1,82 @@ +package rpc_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" +) + +var _ = Describe("RPC transaction converter", func() { + It("converts hex fields to integers", func() { + converter := rpc.RpcTransactionConverter{} + rpcTransaction := getFakeRpcTransaction("0x1") + + transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(transactionModels)).To(Equal(1)) + Expect(transactionModels[0].GasLimit).To(Equal(uint64(1))) + Expect(transactionModels[0].GasPrice).To(Equal(int64(1))) + Expect(transactionModels[0].Nonce).To(Equal(uint64(1))) + Expect(transactionModels[0].TxIndex).To(Equal(int64(1))) + Expect(transactionModels[0].Value).To(Equal("1")) + }) + + It("returns error if invalid hex cannot be converted", func() { + converter := rpc.RpcTransactionConverter{} + invalidTransaction := getFakeRpcTransaction("invalid") + + _, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{invalidTransaction}) + + Expect(err).To(HaveOccurred()) + }) + + It("copies RPC transaction hash, from, and to values to model", func() { + converter := rpc.RpcTransactionConverter{} + rpcTransaction := getFakeRpcTransaction("0x1") + + transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(transactionModels)).To(Equal(1)) + Expect(transactionModels[0].Hash).To(Equal(rpcTransaction.Hash)) + Expect(transactionModels[0].From).To(Equal(rpcTransaction.From)) + Expect(transactionModels[0].To).To(Equal(rpcTransaction.Recipient)) + }) + + XIt("derives transaction RLP", func() { + // actual transaction: https://kovan.etherscan.io/tx/0x73aefdf70fc5650e0dd82affbb59d107f12dfabc50a78625b434ea68b7a69ee6 + // actual RLP hex: 0x2926af093b6b72e3f10089bde6da0f99b0d4e13354f6f37c8334efc9d7e99a47 + + }) + + It("does not include transaction receipt", func() { + converter := rpc.RpcTransactionConverter{} + rpcTransaction := getFakeRpcTransaction("0x1") + + transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(transactionModels)).To(Equal(1)) + Expect(transactionModels[0].Receipt).To(Equal(core.Receipt{})) + }) +}) + +func getFakeRpcTransaction(hex string) core.RpcTransaction { + return core.RpcTransaction{ + Hash: "0x2", + Amount: hex, + GasLimit: hex, + GasPrice: hex, + Nonce: hex, + From: fakes.FakeAddress.Hex(), + Recipient: fakes.FakeAddress.Hex(), + V: "0x2", + R: "0x2", + S: "0x2", + Payload: nil, + TransactionIndex: hex, + } +} From 81dfd12665982d996698ece9a00502b1a0fc1938 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 27 Mar 2019 12:26:04 -0500 Subject: [PATCH 4/5] Address PR comments --- libraries/shared/transactions/syncer.go | 8 +-- libraries/shared/transactions/syncer_test.go | 33 ++++------ pkg/datastore/errors.go | 8 +-- .../repositories/header_repository.go | 15 +++-- .../repositories/header_repository_test.go | 65 +++++++++---------- pkg/datastore/repository.go | 2 +- pkg/fakes/mock_header_repository.go | 10 +-- .../converters/rpc/transaction_converter.go | 2 +- 8 files changed, 69 insertions(+), 74 deletions(-) diff --git a/libraries/shared/transactions/syncer.go b/libraries/shared/transactions/syncer.go index 5f3085c7..8e043dd7 100644 --- a/libraries/shared/transactions/syncer.go +++ b/libraries/shared/transactions/syncer.go @@ -32,11 +32,9 @@ func (syncer TransactionsSyncer) SyncTransactions(headerID int64, logs []types.L if transactionErr != nil { return transactionErr } - for _, transaction := range transactions { - writeErr := syncer.Repository.CreateTransaction(headerID, transaction) - if writeErr != nil { - return writeErr - } + writeErr := syncer.Repository.CreateTransactions(headerID, transactions) + if writeErr != nil { + return writeErr } return nil } diff --git a/libraries/shared/transactions/syncer_test.go b/libraries/shared/transactions/syncer_test.go index 954d2a0b..53df7817 100644 --- a/libraries/shared/transactions/syncer_test.go +++ b/libraries/shared/transactions/syncer_test.go @@ -11,11 +11,19 @@ import ( ) var _ = Describe("Transaction syncer", func() { - It("fetches transactions for logs", func() { - db := test_config.NewTestDB(test_config.NewTestNode()) - blockChain := fakes.NewMockBlockChain() - syncer := transactions.NewTransactionsSyncer(db, blockChain) + var ( + blockChain *fakes.MockBlockChain + syncer transactions.TransactionsSyncer + ) + BeforeEach(func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + blockChain = fakes.NewMockBlockChain() + syncer = transactions.NewTransactionsSyncer(db, blockChain) + }) + + It("fetches transactions for logs", func() { err := syncer.SyncTransactions(0, []types.Log{}) Expect(err).NotTo(HaveOccurred()) @@ -23,10 +31,6 @@ var _ = Describe("Transaction syncer", func() { }) It("only fetches transactions with unique hashes", func() { - db := test_config.NewTestDB(test_config.NewTestNode()) - blockChain := fakes.NewMockBlockChain() - syncer := transactions.NewTransactionsSyncer(db, blockChain) - err := syncer.SyncTransactions(0, []types.Log{{ TxHash: fakes.FakeHash, }, { @@ -38,10 +42,7 @@ var _ = Describe("Transaction syncer", func() { }) It("returns error if fetching transactions fails", func() { - db := test_config.NewTestDB(test_config.NewTestNode()) - blockChain := fakes.NewMockBlockChain() blockChain.GetTransactionsError = fakes.FakeError - syncer := transactions.NewTransactionsSyncer(db, blockChain) err := syncer.SyncTransactions(0, []types.Log{}) @@ -50,26 +51,20 @@ var _ = Describe("Transaction syncer", func() { }) It("passes transactions to repository for persistence", func() { - db := test_config.NewTestDB(test_config.NewTestNode()) - blockChain := fakes.NewMockBlockChain() blockChain.Transactions = []core.TransactionModel{{}} - syncer := transactions.NewTransactionsSyncer(db, blockChain) mockHeaderRepository := fakes.NewMockHeaderRepository() syncer.Repository = mockHeaderRepository err := syncer.SyncTransactions(0, []types.Log{}) Expect(err).NotTo(HaveOccurred()) - Expect(mockHeaderRepository.CreateTransactionCalled).To(BeTrue()) + Expect(mockHeaderRepository.CreateTransactionsCalled).To(BeTrue()) }) It("returns error if persisting transactions fails", func() { - db := test_config.NewTestDB(test_config.NewTestNode()) - blockChain := fakes.NewMockBlockChain() blockChain.Transactions = []core.TransactionModel{{}} - syncer := transactions.NewTransactionsSyncer(db, blockChain) mockHeaderRepository := fakes.NewMockHeaderRepository() - mockHeaderRepository.CreateTransactionError = fakes.FakeError + mockHeaderRepository.CreateTransactionsError = fakes.FakeError syncer.Repository = mockHeaderRepository err := syncer.SyncTransactions(0, []types.Log{}) diff --git a/pkg/datastore/errors.go b/pkg/datastore/errors.go index ec734427..97da92e5 100644 --- a/pkg/datastore/errors.go +++ b/pkg/datastore/errors.go @@ -2,18 +2,18 @@ package datastore import "fmt" -var ErrBlockDoesNotExist = func(blockNumber int64) error { +func ErrBlockDoesNotExist(blockNumber int64) error { return fmt.Errorf("Block number %d does not exist", blockNumber) } -var ErrContractDoesNotExist = func(contractHash string) error { +func ErrContractDoesNotExist(contractHash string) error { return fmt.Errorf("Contract %v does not exist", contractHash) } -var ErrFilterDoesNotExist = func(name string) error { +func ErrFilterDoesNotExist(name string) error { return fmt.Errorf("filter %s does not exist", name) } -var ErrReceiptDoesNotExist = func(txHash string) error { +func ErrReceiptDoesNotExist(txHash string) error { return fmt.Errorf("Receipt for tx: %v does not exist", txHash) } diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index f2235365..6ee990f8 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -49,14 +49,19 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int return 0, ErrValidHeaderExists } -func (repository HeaderRepository) CreateTransaction(headerID int64, transaction core.TransactionModel) error { - _, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions +func (repository HeaderRepository) CreateTransactions(headerID int64, transactions []core.TransactionModel) error { + for _, transaction := range transactions { + _, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions (header_id, hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC) ON CONFLICT DO NOTHING`, headerID, transaction.Hash, transaction.GasLimit, transaction.GasPrice, - transaction.Data, transaction.Nonce, transaction.Raw, transaction.From, transaction.TxIndex, transaction.To, - transaction.Value) - return err + transaction.Data, transaction.Nonce, transaction.Raw, transaction.From, transaction.TxIndex, transaction.To, + transaction.Value) + if err != nil { + return err + } + } + return nil } func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { diff --git a/pkg/datastore/postgres/repositories/header_repository_test.go b/pkg/datastore/postgres/repositories/header_repository_test.go index ef491306..a22f6131 100644 --- a/pkg/datastore/postgres/repositories/header_repository_test.go +++ b/pkg/datastore/postgres/repositories/header_repository_test.go @@ -181,14 +181,21 @@ var _ = Describe("Block header repository", func() { }) Describe("creating a transaction", func() { - It("adds a transaction", func() { - headerID, err := repo.CreateOrUpdateHeader(header) + var ( + headerID int64 + transactions []core.TransactionModel + ) + + BeforeEach(func() { + var err error + headerID, err = repo.CreateOrUpdateHeader(header) Expect(err).NotTo(HaveOccurred()) fromAddress := common.HexToAddress("0x1234") toAddress := common.HexToAddress("0x5678") txHash := common.HexToHash("0x9876") + txHashTwo := common.HexToHash("0x5432") txIndex := big.NewInt(123) - transaction := core.TransactionModel{ + transactions = []core.TransactionModel{{ Data: []byte{}, From: fromAddress.Hex(), GasLimit: 0, @@ -199,44 +206,34 @@ var _ = Describe("Block header repository", func() { To: toAddress.Hex(), TxIndex: txIndex.Int64(), Value: "0", - } - - insertErr := repo.CreateTransaction(headerID, transaction) + }, { + Data: []byte{}, + From: fromAddress.Hex(), + GasLimit: 1, + GasPrice: 1, + Hash: txHashTwo.Hex(), + Nonce: 1, + Raw: []byte{}, + To: toAddress.Hex(), + TxIndex: 1, + Value: "1", + }} + insertErr := repo.CreateTransactions(headerID, transactions) Expect(insertErr).NotTo(HaveOccurred()) - var dbTransaction core.TransactionModel - err = db.Get(&dbTransaction, + }) + + It("adds transactions", func() { + var dbTransactions []core.TransactionModel + err = db.Select(&dbTransactions, `SELECT hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value" FROM public.light_sync_transactions WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(dbTransaction).To(Equal(transaction)) + Expect(dbTransactions).To(ConsistOf(transactions)) }) It("silently ignores duplicate inserts", func() { - headerID, err := repo.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - fromAddress := common.HexToAddress("0x1234") - toAddress := common.HexToAddress("0x5678") - txHash := common.HexToHash("0x9876") - txIndex := big.NewInt(123) - transaction := core.TransactionModel{ - Data: []byte{}, - From: fromAddress.Hex(), - GasLimit: 0, - GasPrice: 0, - Hash: txHash.Hex(), - Nonce: 0, - Raw: []byte{}, - Receipt: core.Receipt{}, - To: toAddress.Hex(), - TxIndex: txIndex.Int64(), - Value: "0", - } - - insertErr := repo.CreateTransaction(headerID, transaction) - Expect(insertErr).NotTo(HaveOccurred()) - - insertTwoErr := repo.CreateTransaction(headerID, transaction) + insertTwoErr := repo.CreateTransactions(headerID, transactions) Expect(insertTwoErr).NotTo(HaveOccurred()) var dbTransactions []core.TransactionModel @@ -244,7 +241,7 @@ var _ = Describe("Block header repository", func() { `SELECT hash, gaslimit, gasprice, input_data, nonce, raw, tx_from, tx_index, tx_to, "value" FROM public.light_sync_transactions WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - Expect(len(dbTransactions)).To(Equal(1)) + Expect(len(dbTransactions)).To(Equal(2)) }) }) diff --git a/pkg/datastore/repository.go b/pkg/datastore/repository.go index 0b5af3dd..db0c220b 100644 --- a/pkg/datastore/repository.go +++ b/pkg/datastore/repository.go @@ -41,7 +41,7 @@ type FilterRepository interface { type HeaderRepository interface { CreateOrUpdateHeader(header core.Header) (int64, error) - CreateTransaction(headerID int64, transaction core.TransactionModel) error + CreateTransactions(headerID int64, transactions []core.TransactionModel) error GetHeader(blockNumber int64) (core.Header, error) MissingBlockNumbers(startingBlockNumber, endingBlockNumber int64, nodeID string) ([]int64, error) } diff --git a/pkg/fakes/mock_header_repository.go b/pkg/fakes/mock_header_repository.go index c89d8f16..8f268146 100644 --- a/pkg/fakes/mock_header_repository.go +++ b/pkg/fakes/mock_header_repository.go @@ -27,8 +27,8 @@ type MockHeaderRepository struct { createOrUpdateHeaderErr error createOrUpdateHeaderPassedBlockNumbers []int64 createOrUpdateHeaderReturnID int64 - CreateTransactionCalled bool - CreateTransactionError error + CreateTransactionsCalled bool + CreateTransactionsError error getHeaderError error getHeaderReturnBlockHash string missingBlockNumbers []int64 @@ -58,9 +58,9 @@ func (repository *MockHeaderRepository) CreateOrUpdateHeader(header core.Header) return repository.createOrUpdateHeaderReturnID, repository.createOrUpdateHeaderErr } -func (repository *MockHeaderRepository) CreateTransaction(headerID int64, transaction core.TransactionModel) error { - repository.CreateTransactionCalled = true - return repository.CreateTransactionError +func (repository *MockHeaderRepository) CreateTransactions(headerID int64, transactions []core.TransactionModel) error { + repository.CreateTransactionsCalled = true + return repository.CreateTransactionsError } func (repository *MockHeaderRepository) GetHeader(blockNumber int64) (core.Header, error) { diff --git a/pkg/geth/converters/rpc/transaction_converter.go b/pkg/geth/converters/rpc/transaction_converter.go index 067a1e99..1c06ac87 100644 --- a/pkg/geth/converters/rpc/transaction_converter.go +++ b/pkg/geth/converters/rpc/transaction_converter.go @@ -25,8 +25,8 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" "golang.org/x/sync/errgroup" "github.com/vulcanize/vulcanizedb/pkg/core" From f5b32a11b0256dceaedee3f328780a3e8e65a514 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Wed, 27 Mar 2019 13:49:44 -0500 Subject: [PATCH 5/5] Improve transaction syncing test coverage --- integration_test/geth_blockchain_test.go | 36 ++++++++++++++++ pkg/core/transaction.go | 2 +- .../converters/rpc/transaction_converter.go | 5 ++- .../rpc/transaction_converter_test.go | 42 +++++++++++++++---- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/integration_test/geth_blockchain_test.go b/integration_test/geth_blockchain_test.go index c970c799..69b15a82 100644 --- a/integration_test/geth_blockchain_test.go +++ b/integration_test/geth_blockchain_test.go @@ -17,6 +17,7 @@ package integration_test import ( + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" . "github.com/onsi/ginkgo" @@ -85,6 +86,41 @@ var _ = Describe("Reading from the Geth blockchain", func() { close(done) }, 15) + It("retrieves transaction", func() { + // actual transaction: https://etherscan.io/tx/0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255 + txHash := common.HexToHash("0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255") + transactions, err := blockChain.GetTransactions([]common.Hash{txHash}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(transactions)).To(Equal(1)) + expectedData := []byte{149, 227, 197, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 160, 85, 105, 13, 157, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, + 241, 202, 218, 90, 30, 178, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 92, 155, 193, 43} + expectedRaw := []byte{248, 201, 9, 132, 59, 154, 202, 0, 131, 1, 102, 93, 148, 44, 75, 208, 100, 185, 152, 131, + 128, 118, 250, 52, 26, 131, 208, 7, 252, 47, 165, 9, 87, 128, 184, 100, 149, 227, 197, 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 160, 85, 105, 13, 157, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 241, 202, 218, 90, 30, 178, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 155, 193, 43, 37, 160, 237, 184, 236, 248, 23, 152, + 53, 238, 44, 215, 181, 234, 229, 157, 246, 212, 178, 88, 25, 116, 134, 163, 124, 64, 2, 66, 25, 118, 1, 253, 27, + 101, 160, 36, 226, 116, 43, 147, 236, 124, 76, 227, 250, 228, 168, 22, 19, 248, 155, 248, 151, 219, 14, 1, 186, + 159, 35, 154, 22, 222, 123, 254, 147, 63, 221} + expectedModel := core.TransactionModel{ + Data: expectedData, + From: "0x3b08b99441086edd66f36f9f9aee733280698378", + GasLimit: 91741, + GasPrice: 1000000000, + Hash: "0x44d462f2a19ad267e276b234a62c542fc91c974d2e4754a325ca405f95440255", + Nonce: 9, + Raw: expectedRaw, + Receipt: core.Receipt{}, + To: "0x2c4bd064b998838076fa341a83d007fc2fa50957", + TxIndex: 30, + Value: "0", + } + Expect(transactions[0]).To(Equal(expectedModel)) + }) + //Benchmarking test: remove skip to test performance of block retrieval XMeasure("retrieving n blocks", func(b Benchmarker) { b.Time("runtime", func() { diff --git a/pkg/core/transaction.go b/pkg/core/transaction.go index f9c0515b..c5198cf7 100644 --- a/pkg/core/transaction.go +++ b/pkg/core/transaction.go @@ -36,7 +36,7 @@ type RpcTransaction struct { GasLimit string `json:"gas"` Recipient string `json:"to"` Amount string `json:"value"` - Payload []byte `json:"input"` + Payload string `json:"input"` V string `json:"v"` R string `json:"r"` S string `json:"s"` diff --git a/pkg/geth/converters/rpc/transaction_converter.go b/pkg/geth/converters/rpc/transaction_converter.go index 1c06ac87..7906ec41 100644 --- a/pkg/geth/converters/rpc/transaction_converter.go +++ b/pkg/geth/converters/rpc/transaction_converter.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "fmt" + "github.com/ethereum/go-ethereum/common/hexutil" "log" "math/big" "strings" @@ -70,7 +71,7 @@ func (converter *RpcTransactionConverter) ConvertRpcTransactionsToModels(transac return nil, txIndexErr } transactionModel := core.TransactionModel{ - Data: transaction.Payload, + Data: txData.Payload, From: transaction.From, GasLimit: txData.GasLimit, GasPrice: txData.Price.Int64(), @@ -187,7 +188,7 @@ func getTransactionData(transaction core.RpcTransaction) (transactionData, error GasLimit: gasLimit.Uint64(), Recipient: &recipient, Amount: amount, - Payload: transaction.Payload, + Payload: hexutil.MustDecode(transaction.Payload), V: v, R: r, S: s, diff --git a/pkg/geth/converters/rpc/transaction_converter_test.go b/pkg/geth/converters/rpc/transaction_converter_test.go index 705e0572..21047c92 100644 --- a/pkg/geth/converters/rpc/transaction_converter_test.go +++ b/pkg/geth/converters/rpc/transaction_converter_test.go @@ -9,8 +9,13 @@ import ( ) var _ = Describe("RPC transaction converter", func() { + var converter rpc.RpcTransactionConverter + + BeforeEach(func() { + converter = rpc.RpcTransactionConverter{} + }) + It("converts hex fields to integers", func() { - converter := rpc.RpcTransactionConverter{} rpcTransaction := getFakeRpcTransaction("0x1") transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) @@ -25,7 +30,6 @@ var _ = Describe("RPC transaction converter", func() { }) It("returns error if invalid hex cannot be converted", func() { - converter := rpc.RpcTransactionConverter{} invalidTransaction := getFakeRpcTransaction("invalid") _, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{invalidTransaction}) @@ -34,7 +38,6 @@ var _ = Describe("RPC transaction converter", func() { }) It("copies RPC transaction hash, from, and to values to model", func() { - converter := rpc.RpcTransactionConverter{} rpcTransaction := getFakeRpcTransaction("0x1") transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) @@ -46,14 +49,37 @@ var _ = Describe("RPC transaction converter", func() { Expect(transactionModels[0].To).To(Equal(rpcTransaction.Recipient)) }) - XIt("derives transaction RLP", func() { - // actual transaction: https://kovan.etherscan.io/tx/0x73aefdf70fc5650e0dd82affbb59d107f12dfabc50a78625b434ea68b7a69ee6 - // actual RLP hex: 0x2926af093b6b72e3f10089bde6da0f99b0d4e13354f6f37c8334efc9d7e99a47 + It("derives transaction RLP", func() { + // actual transaction: https://kovan.etherscan.io/tx/0x3b29ef265425d304069c57e5145cd1c7558568b06d231775f50a693bee1aad4f + rpcTransaction := core.RpcTransaction{ + Nonce: "0x7aa9", + GasPrice: "0x3b9aca00", + GasLimit: "0x7a120", + Recipient: "0xf88bbdc1e2718f8857f30a180076ec38d53cf296", + Amount: "0x0", + Payload: "0x18178358", + V: "0x78", + R: "0x79f6a78ababfdb37b87a4d52795a49b08b5b5171443d1f2fb8f373431e77439c", + S: "0x3f1a210dd3b59d161735a314b88568fa91552dfe207c00a2fdbcd52ccb081409", + Hash: "0x3b29ef265425d304069c57e5145cd1c7558568b06d231775f50a693bee1aad4f", + From: "0x694032e172d9b0ee6aff5d36749bad4947a36e4e", + TransactionIndex: "0xa", + } + transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(transactionModels)).To(Equal(1)) + model := transactionModels[0] + expectedRLP := []byte{248, 106, 130, 122, 169, 132, 59, 154, 202, 0, 131, 7, 161, 32, 148, 248, 139, 189, 193, + 226, 113, 143, 136, 87, 243, 10, 24, 0, 118, 236, 56, 213, 60, 242, 150, 128, 132, 24, 23, 131, 88, 120, 160, + 121, 246, 167, 138, 186, 191, 219, 55, 184, 122, 77, 82, 121, 90, 73, 176, 139, 91, 81, 113, 68, 61, 31, 47, + 184, 243, 115, 67, 30, 119, 67, 156, 160, 63, 26, 33, 13, 211, 181, 157, 22, 23, 53, 163, 20, 184, 133, 104, + 250, 145, 85, 45, 254, 32, 124, 0, 162, 253, 188, 213, 44, 203, 8, 20, 9} + Expect(model.Raw).To(Equal(expectedRLP)) }) It("does not include transaction receipt", func() { - converter := rpc.RpcTransactionConverter{} rpcTransaction := getFakeRpcTransaction("0x1") transactionModels, err := converter.ConvertRpcTransactionsToModels([]core.RpcTransaction{rpcTransaction}) @@ -76,7 +102,7 @@ func getFakeRpcTransaction(hex string) core.RpcTransaction { V: "0x2", R: "0x2", S: "0x2", - Payload: nil, + Payload: "0x12", TransactionIndex: hex, } }