From cf437e40077028b23be5746a32dab312514bcdce Mon Sep 17 00:00:00 2001 From: David Terry Date: Fri, 28 Sep 2018 16:08:06 +0200 Subject: [PATCH 01/30] VDB-101: Add signature for vat.fold --- pkg/transformers/shared/constants.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index fd3f03d3..74983e92 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -56,6 +56,7 @@ var ( pitFileStabilityFeeMethod = GetSolidityMethodSignature(PitABI, "file") tendMethod = GetSolidityMethodSignature(FlipperABI, "tend") vatInitMethod = GetSolidityMethodSignature(VatABI, "init") + vatFoldMethod = GetSolidityMethodSignature(VatABI, "fold") BiteSignature = GetEventSignature(biteMethod) DealSignature = GetLogNoteSignature(dealMethod) @@ -76,4 +77,5 @@ var ( PitFileStabilityFeeSignature = GetLogNoteSignature(pitFileStabilityFeeMethod) TendFunctionSignature = GetLogNoteSignature(tendMethod) VatInitSignature = GetLogNoteSignature(vatInitMethod) + VatFoldSignature = GetLogNoteSignature(vatFoldMethod) ) From 377c57ce1b384a5e45130f7ed121f3748a122e93 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 12:44:05 +0200 Subject: [PATCH 02/30] transformers: format documentation --- pkg/transformers/DOCUMENTATION.md | 172 +++++++++++++++++++----------- 1 file changed, 107 insertions(+), 65 deletions(-) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index f46f0ee5..40837e77 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -1,70 +1,112 @@ -The main goal of creating a transformer is to fetch specific log events from Ethereum, convert/decode them into usable data and then persist them to VulcanizeDB. For Maker there are two main types of log events that we're tracking: custom events that are defined in the contract solidity code, and LogNote events which utilize the [DSNote library](https://github.com/dapphub/ds-note). The transformer process for each of these different log types is the same, except for the converting process, as denoted below. +# Transformers -To illustrate how to create a custom log event transformer we'll use the Kick event defined in [flop.sol](https://github.com/makerdao/dss/blob/master/src/flop.sol) as an example. +## Architecture -1. Get an example raw log event either from mainnet (if the contract has already been deployed), from the Kovan testnet, or by deploying the contract to a local chain and emitting the event manually. We will use the example log event to test drive converting the log to a database model. +Transformers fetch logs from Ethereum, convert/decode them into usable data, and then persist them in postgres. -1. Fetch the logs from the chain based on log event's topic zero. - - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in `pkg/transformers/shared/constants.go`. - - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. - - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. +A transformer consists of: -1. Convert the raw log into a database model. - - **Converting most custom events** (such as FlopKick) - 1. Convert the raw log into a Go struct. - - We've been using [go-ethereum's abigen tool](https://github.com/ethereum/go-ethereum/tree/master/cmd/abigen) to get the contract's ABI, and a Go struct that represents the event log. We will unpack the raw logs into this struct. - - To use abigen: `abigen --sol flip.sol --pkg flip --out {/path/to/output_file}` - - sol: this is the path to the solidity contract - - pkg: a package name for the generated Go code - - out: the file path for the generated Go code (optional) - - the output for `flop.sol` will include the FlopperAbi and the FlopperKick struct: - ```go - type FlopperKick struct { - Id *big.Int - Lot *big.Int - Bid *big.Int - Gal common.Address - End *big.Int - Raw types.Log - } - ``` - - Using go-ethereum's `contract.UnpackLog` method we can unpack the raw log into the FlopperKick struct (which we're referring to as the `entity`). - - See the `ToEntity` method in `pkg/transformers/flop_kick/converter`. - - The unpack method will not add the `Raw` or `TransactionIndex` values to the entity struct - both of these values are accessible from the raw types.Log. - 1. Convert the entity into a database model. See the `ToModel` method in `pkg/transformers/flop_kick/converter`. +- A fetcher -> Fetches raw logs from the blockchain encoded as go datatypes +- A converter -> Converts this raw data into a representation suitable for consumption in the API +- A repository -> Abstracts the database - - **Converting Price Feed custom events** - - Price Feed contracts use the [LogNote event](https://github.com/makerdao/medianizer/blob/master/src/medianizer.sol#L23) - - The LogNote event takes in the value of the price feed as it's sole argument, and does not index it. This means that this value can be taken directly from the log's data, and then properly converted using the `price_feeds.Convert` method (located in the model.go file). - - Since this conversion from raw log to model includes less fields than some others, we've chosen to convert it directly to the database model, skipping the `ToEntity` step. - - **Converting LogNote events** (such as tend) - - Since LogNote events are a generic structure, they depend on the method signature of the method that is calling them. For example, the `tend` method is called on the [flip.sol contract](https://github.com/makerdao/dss/blob/master/src/flip.sol#L117), and it's method signature looks like this: `tend(uint,uint,uint)`. - - The first four bytes of the Keccak-256 hashed method signature will be located in `topic[0]` on the log. - - The message sender will be in `topic[1]`. - - The first parameter passed to `tend` becomes `topic[2]`. - - The second parameter passed to `tend` will be `topic[3]`. - - Any additional parameters will be in the log's data field. - - More detail is located in the [DSNote repo](https://github.com/dapphub/ds-note). -1. Get all MissingHeaders: - - Headers are inserted into VulcanizeDB as part of the `lightSync` command. Then for each transformer we check each header for matching logs. - - The MissingHeaders method queries the `checked_headers` table to see if the header has been checked for the given log type. -1. Persist the log record to VulcanizeDB. - - Each event log has it's own table in the database, as well as it's own column in the `checked_headers` table. - - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type. - - To create a new migration file: `./scripts/create_migration create_flop_kick` - - See `db/migrations/1536942529_create_flop_kick.up.sql`. - - The specific log event tables are all created in the `maker` - schema. - - There is a one-many association between `headers` and the log - event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed. - - To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private` - - When a new log record is inserted into VulcanizeDB, we also need to make sure to insert a record into the `checked_headers` table for the given log type. - - We have been using the repository pattern (i.e. wrapping all SQL/ORM invocations in isolated namespaces per table) to interact with the database, see the `Create` method in `pkg/transformers/flop_kick/repository.go`. -1. MarkHeaderChecked: - - There is a chance that a header does not have a log for the given transformer's log type, and in this instance we also want to record that the header has been "checked" so that we don't continue to query that header over and over. - - In the transformer we'll make sure to insert a row for the header indicating that it has been checked for the log type that the transformer is responsible for. -1. Wire each component up in the transformer. - - We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`. - - See any of `pkg/transformers/flop_kick/transformer.go` - - All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration. - - The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers. +For Maker, vulcanize will be run in `lightSync` mode, so it will store all headers, and then fetchers make RPC calls to pull th + +## Event Types + +For Maker there are two main types of log events that we're tracking: + +1. Custom events that are defined in the contract solidity code. +1. LogNote events which utilize the [DSNote library](https://github.com/dapphub/ds-note). + +The transformer process for each of these different log types is the same, except for the converting process, as denoted below. + +## Creating a Transformer + +**Fetching Logs** + +1. Generate an example raw log event, by either: + + - Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)). + - Deploying the contract to a local chain and emiting the event manually. + +1. Fetch the logs from the chain based on the example event's topic zero: + + - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in `pkg/transformers/shared/constants.go`. + - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. + - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. + +**Coverting logs** + +- **Converting most custom events** (such as FlopKick) + + 1. Convert the raw log into a Go struct. + - We've been using [go-ethereum's abigen tool](https://github.com/ethereum/go-ethereum/tree/master/cmd/abigen) to get the contract's ABI, and a Go struct that represents the event log. We will unpack the raw logs into this struct. + - To use abigen: `abigen --sol flip.sol --pkg flip --out {/path/to/output_file}` + - sol: this is the path to the solidity contract + - pkg: a package name for the generated Go code + - out: the file path for the generated Go code (optional) + - the output for `flop.sol` will include the FlopperAbi and the FlopperKick struct: + ```go + type FlopperKick struct { + Id *big.Int + Lot *big.Int + Bid *big.Int + Gal common.Address + End *big.Int + Raw types.Log + } + ``` + - Using go-ethereum's `contract.UnpackLog` method we can unpack the raw log into the FlopperKick struct (which we're referring to as the `entity`). + - See the `ToEntity` method in `pkg/transformers/flop_kick/converter`. + 1. Convert the entity into a database model. See the `ToModel` method in `pkg/transformers/flop_kick/converter`. + +- **Converting Price Feed custom events** + + - Price Feed contracts use the [LogNote event](https://github.com/makerdao/medianizer/blob/master/src/medianizer.sol#L23) + - The LogNote event takes in the value of the price feed as it's sole argument, and does not index it. This means that this value can be taken directly from the log's data, and then properly converted using the `price_feeds.Convert` method (located in the model.go file). + - Since this conversion from raw log to model includes less fields than some others, we've chosen to convert it directly to the database model, skipping the `ToEntity` step. + +- **Converting LogNote events** (such as tend) + - Since LogNote events are a generic structure, they depend on the method signature of the method that is calling them. For example, the `tend` method is called on the [flip.sol contract](https://github.com/makerdao/dss/blob/master/src/flip.sol#L117), and it's method signature looks like this: `tend(uint,uint,uint)`. + - The first four bytes of the Keccak-256 hashed method signature will be located in `topic[0]` on the log. + - The message sender will be in `topic[1]`. + - The first parameter passed to `tend` becomes `topic[2]`. + - The second parameter passed to `tend` will be `topic[3]`. + - Any additional parameters will be in the log's data field. + - More detail is located in the [DSNote repo](https://github.com/dapphub/ds-note). + +**Get all MissingHeaders** + +- Headers are inserted into VulcanizeDB as part of the `lightSync` command. Then for each transformer we check each header for matching logs. +- The MissingHeaders method queries the `checked_headers` table to see if the header has been checked for the given log type. + +**Persist the log record to VulcanizeDB** + +- Each event log has it's own table in the database, as well as it's own column in the `checked_headers` table. + - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type. +- To create a new migration file: `./scripts/create_migration create_flop_kick` + - See `db/migrations/1536942529_create_flop_kick.up.sql`. + - The specific log event tables are all created in the `maker` + schema. + - There is a one-many association between `headers` and the log + event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed. +- To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private` +- When a new log record is inserted into VulcanizeDB, we also need to make sure to insert a record into the `checked_headers` table for the given log type. +- We have been using the repository pattern (i.e. wrapping all SQL/ORM invocations in isolated namespaces per table) to interact with the database, see the `Create` method in `pkg/transformers/flop_kick/repository.go`. + +**MarkHeaderChecked** + +- There is a chance that a header does not have a log for the given transformer's log type, and in this instance we also want to record that the header has been "checked" so that we don't continue to query that header over and over. +- In the transformer we'll make sure to insert a row for the header indicating that it has been checked for the log type that the transformer is responsible for. + +**Wire each component up in the transformer** + +- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`. +- See any of `pkg/transformers/flop_kick/transformer.go` +- All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration. +- The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers. + +## Useful Documents + +[Ethereum Event ABI Specification](https://solidity.readthedocs.io/en/develop/abi-spec.html#events) From 2feb7ea546371a3942ad6d0794089dbe300f64c5 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 12:44:23 +0200 Subject: [PATCH 03/30] Vat.fold: add migrations --- .../1538510582_create_vat_fold_table.down.sql | 3 + .../1538510582_create_vat_fold_table.up.sql | 13 ++++ db/schema.sql | 73 ++++++++++++++++++- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 db/migrations/1538510582_create_vat_fold_table.down.sql create mode 100644 db/migrations/1538510582_create_vat_fold_table.up.sql diff --git a/db/migrations/1538510582_create_vat_fold_table.down.sql b/db/migrations/1538510582_create_vat_fold_table.down.sql new file mode 100644 index 00000000..bfbedda7 --- /dev/null +++ b/db/migrations/1538510582_create_vat_fold_table.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.flop_kick; +ALTER TABLE public.checked_headers + DROP COLUMN flop_kick_checked; diff --git a/db/migrations/1538510582_create_vat_fold_table.up.sql b/db/migrations/1538510582_create_vat_fold_table.up.sql new file mode 100644 index 00000000..327719e1 --- /dev/null +++ b/db/migrations/1538510582_create_vat_fold_table.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE maker.vat_fold ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk bytea, + urn bytea, + rate numeric, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_fold_checked BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/db/schema.sql b/db/schema.sql index 4a00afc8..83e79100 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10.3 --- Dumped by pg_dump version 10.3 +-- Dumped from database version 10.5 +-- Dumped by pg_dump version 10.5 SET statement_timeout = 0; SET lock_timeout = 0; @@ -697,6 +697,41 @@ CREATE SEQUENCE maker.tend_id_seq ALTER SEQUENCE maker.tend_id_seq OWNED BY maker.tend.id; +-- +-- Name: vat_fold; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_fold ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk bytea, + urn bytea, + rate numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_fold_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_fold_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_fold_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_fold_id_seq OWNED BY maker.vat_fold.id; + + -- -- Name: vat_init; Type: TABLE; Schema: maker; Owner: - -- @@ -820,7 +855,8 @@ CREATE TABLE public.checked_headers ( tend_checked boolean DEFAULT false NOT NULL, cat_file_chop_lump_checked boolean DEFAULT false NOT NULL, cat_file_flip_checked boolean DEFAULT false NOT NULL, - cat_file_pit_vow_checked boolean DEFAULT false NOT NULL + cat_file_pit_vow_checked boolean DEFAULT false NOT NULL, + vat_fold_checked boolean DEFAULT false NOT NULL ); @@ -1267,6 +1303,13 @@ ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.pr ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_seq'::regclass); +-- +-- Name: vat_fold id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_fold ALTER COLUMN id SET DEFAULT nextval('maker.vat_fold_id_seq'::regclass); + + -- -- Name: vat_init id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1648,6 +1691,22 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_pkey PRIMARY KEY (id); +-- +-- Name: vat_fold vat_fold_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_fold + ADD CONSTRAINT vat_fold_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_fold vat_fold_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_fold + ADD CONSTRAINT vat_fold_pkey PRIMARY KEY (id); + + -- -- Name: vat_init vat_init_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -1962,6 +2021,14 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_fold vat_fold_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_fold + ADD CONSTRAINT vat_fold_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: vat_init vat_init_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- From 22fb019d3f80c9aa4eb55935635c0f672df9f276 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 12:48:23 +0200 Subject: [PATCH 04/30] docs: add links --- pkg/transformers/DOCUMENTATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index 40837e77..2abc031f 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -32,7 +32,7 @@ The transformer process for each of these different log types is the same, excep 1. Fetch the logs from the chain based on the example event's topic zero: - - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in `pkg/transformers/shared/constants.go`. + - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go). - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. @@ -58,7 +58,7 @@ The transformer process for each of these different log types is the same, excep } ``` - Using go-ethereum's `contract.UnpackLog` method we can unpack the raw log into the FlopperKick struct (which we're referring to as the `entity`). - - See the `ToEntity` method in `pkg/transformers/flop_kick/converter`. + - See the `ToEntity` method in [`pkg/transformers/flop_kick/converter.go`](./flop_kick/converter.go). 1. Convert the entity into a database model. See the `ToModel` method in `pkg/transformers/flop_kick/converter`. - **Converting Price Feed custom events** From ce4d928fb84c29a9da37a806580f2e29fa945ca1 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 14:07:32 +0200 Subject: [PATCH 05/30] transformers: tweak docs --- pkg/transformers/DOCUMENTATION.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index 2abc031f..c53130ad 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -6,11 +6,11 @@ Transformers fetch logs from Ethereum, convert/decode them into usable data, and A transformer consists of: -- A fetcher -> Fetches raw logs from the blockchain encoded as go datatypes -- A converter -> Converts this raw data into a representation suitable for consumption in the API +- A fetcher -> Fetches raw logs from the blockchain and encodes them as go datatypes +- A converter -> Converts this raw data into a human friendly representation suitable for consumption in the API - A repository -> Abstracts the database -For Maker, vulcanize will be run in `lightSync` mode, so it will store all headers, and then fetchers make RPC calls to pull th +For Maker, vulcanize will be run in `lightSync` mode, so it will store all headers, and then fetchers pull relevant logs by making RPC calls. ## Event Types From b3295e3a86e49c0536441d57857b9954f02474a4 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 14:08:01 +0200 Subject: [PATCH 06/30] Vat.fold: add converter & tests --- pkg/transformers/test_data/vat_fold.go | 53 +++++++++++++++++ pkg/transformers/vat_fold/converter.go | 57 +++++++++++++++++++ pkg/transformers/vat_fold/converter_test.go | 44 ++++++++++++++ pkg/transformers/vat_fold/model.go | 9 +++ .../vat_fold/vat_fold_suite_test.go | 19 +++++++ 5 files changed, 182 insertions(+) create mode 100644 pkg/transformers/test_data/vat_fold.go create mode 100644 pkg/transformers/vat_fold/converter.go create mode 100644 pkg/transformers/vat_fold/converter_test.go create mode 100644 pkg/transformers/vat_fold/model.go create mode 100644 pkg/transformers/vat_fold/vat_fold_suite_test.go diff --git a/pkg/transformers/test_data/vat_fold.go b/pkg/transformers/test_data/vat_fold.go new file mode 100644 index 00000000..98334d8b --- /dev/null +++ b/pkg/transformers/test_data/vat_fold.go @@ -0,0 +1,53 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test_data + +import ( + "bytes" + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" +) + +var EthVatFoldLog = types.Log{ + Address: common.HexToAddress(shared.VatContractAddress), + Topics: []common.Hash{ + common.HexToHash("0xe6a6a64d00000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x00000000000000000000000007Fa9eF6609cA7921112231F8f195138ebbA2977"), + common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), + common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002"), + }, + Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000243b66319566616b6520696c6b000000000000000000000000000000000000000000000000"), + BlockNumber: 72, + TxHash: common.HexToHash("0xe8f39fbb7fea3621f543868f19b1114e305aff6a063a30d32835ff1012526f91"), + TxIndex: 8, + BlockHash: common.HexToHash("0xe3dd2e05bd8b92833e20ed83e2171bbc06a9ec823232eca1730a807bd8f5edc0"), + Index: 5, + Removed: false, +} + +var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog) +var VatFoldModel = vat_fold.VatFoldModel{ + Ilk: string(bytes.Trim(EthVatFoldLog.Topics[1].Bytes(), "\x00")), + Urn: string(bytes.Trim(EthVatFoldLog.Topics[2].Bytes(), "\x00")), + Rate: string(bytes.Trim(EthVatFoldLog.Topics[3].Bytes(), "\x00")), + TransactionIndex: EthVatFoldLog.TxIndex, + Raw: rawVatFoldLog, +} diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go new file mode 100644 index 00000000..74fb3b63 --- /dev/null +++ b/pkg/transformers/vat_fold/converter.go @@ -0,0 +1,57 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/core/types" +) + +type Converter interface { + ToModel(ethLog types.Log) (VatFoldModel, error) +} + +type VatFoldConverter struct{} + +func (VatFoldConverter) ToModel(ethLog types.Log) (VatFoldModel, error) { + err := verifyLog(ethLog) + if err != nil { + return VatFoldModel{}, err + } + + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) + rate := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + raw, err := json.Marshal(ethLog) + + return VatFoldModel{ + Ilk: ilk, + Urn: urn, + Rate: rate, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + }, err +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + + return nil +} diff --git a/pkg/transformers/vat_fold/converter_test.go b/pkg/transformers/vat_fold/converter_test.go new file mode 100644 index 00000000..021096db --- /dev/null +++ b/pkg/transformers/vat_fold/converter_test.go @@ -0,0 +1,44 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" +) + +var _ = Describe("Vat fold converter", func() { + It("returns err if log missing topics", func() { + converter := vat_fold.VatFoldConverter{} + badLog := types.Log{} + + _, err := converter.ToModel(badLog) + + Expect(err).To(HaveOccurred()) + }) + + It("converts a log to an model", func() { + converter := vat_fold.VatFoldConverter{} + + model, err := converter.ToModel(test_data.EthVatFoldLog) + + Expect(err).NotTo(HaveOccurred()) + Expect(model).To(Equal(test_data.VatFoldModel)) + }) +}) diff --git a/pkg/transformers/vat_fold/model.go b/pkg/transformers/vat_fold/model.go new file mode 100644 index 00000000..9af973b3 --- /dev/null +++ b/pkg/transformers/vat_fold/model.go @@ -0,0 +1,9 @@ +package vat_fold + +type VatFoldModel struct { + Ilk string + Urn string + Rate string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_fold/vat_fold_suite_test.go b/pkg/transformers/vat_fold/vat_fold_suite_test.go new file mode 100644 index 00000000..97a2aac7 --- /dev/null +++ b/pkg/transformers/vat_fold/vat_fold_suite_test.go @@ -0,0 +1,19 @@ +package vat_fold_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVatFold(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatFold Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) From 96f8279421483542c462816452aa818b9bce2f67 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 16:03:30 +0200 Subject: [PATCH 07/30] Vat.fold: update test data so it matches a real deployed kovan event --- pkg/transformers/test_data/vat_fold.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/transformers/test_data/vat_fold.go b/pkg/transformers/test_data/vat_fold.go index 98334d8b..8a5e9b25 100644 --- a/pkg/transformers/test_data/vat_fold.go +++ b/pkg/transformers/test_data/vat_fold.go @@ -30,11 +30,11 @@ var EthVatFoldLog = types.Log{ Address: common.HexToAddress(shared.VatContractAddress), Topics: []common.Hash{ common.HexToHash("0xe6a6a64d00000000000000000000000000000000000000000000000000000000"), - common.HexToHash("0x00000000000000000000000007Fa9eF6609cA7921112231F8f195138ebbA2977"), - common.HexToHash("0x00000000000000000000000064d922894153be9eef7b7218dc565d1d0ce2a092"), + common.HexToHash("0x5245500000000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x0000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d1"), common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002"), }, - Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000243b66319566616b6520696c6b000000000000000000000000000000000000000000000000"), + Data: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000064e6a6a64d45544800000000000000000000000000000000000000000000000000000000000000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d10000000000000000000000000000000000000000000000000000000000000000"), BlockNumber: 72, TxHash: common.HexToHash("0xe8f39fbb7fea3621f543868f19b1114e305aff6a063a30d32835ff1012526f91"), TxIndex: 8, From 0c58e0ac5b04d79c1329fec62a04ce89d9e96b56 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 18:21:27 +0200 Subject: [PATCH 08/30] Vat.fold: verify that the event signature is correct --- pkg/transformers/vat_fold/converter.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go index 74fb3b63..6270c647 100644 --- a/pkg/transformers/vat_fold/converter.go +++ b/pkg/transformers/vat_fold/converter.go @@ -53,5 +53,10 @@ func verifyLog(log types.Log) error { return errors.New("log missing topics") } + sig := log.Topics[0].String() + if sig != shared.VatFoldSignature { + return errors.New("log is not a Vat.fold event") + } + return nil } From ef0e3f9e117ee2e95bb2895049c0dd96d158404b Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 18:22:05 +0200 Subject: [PATCH 09/30] Vat.fold: add repository & tests --- pkg/transformers/test_data/vat_fold.go | 13 +- pkg/transformers/vat_fold/converter.go | 7 +- pkg/transformers/vat_fold/repository.go | 60 ++++++++ pkg/transformers/vat_fold/repository_test.go | 145 +++++++++++++++++++ 4 files changed, 216 insertions(+), 9 deletions(-) create mode 100644 pkg/transformers/vat_fold/repository.go create mode 100644 pkg/transformers/vat_fold/repository_test.go diff --git a/pkg/transformers/test_data/vat_fold.go b/pkg/transformers/test_data/vat_fold.go index 8a5e9b25..e033da17 100644 --- a/pkg/transformers/test_data/vat_fold.go +++ b/pkg/transformers/test_data/vat_fold.go @@ -15,7 +15,6 @@ package test_data import ( - "bytes" "encoding/json" "github.com/ethereum/go-ethereum/common" @@ -35,19 +34,19 @@ var EthVatFoldLog = types.Log{ common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002"), }, Data: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000064e6a6a64d45544800000000000000000000000000000000000000000000000000000000000000000000000000000000003728e9777b2a0a611ee0f89e00e01044ce4736d10000000000000000000000000000000000000000000000000000000000000000"), - BlockNumber: 72, - TxHash: common.HexToHash("0xe8f39fbb7fea3621f543868f19b1114e305aff6a063a30d32835ff1012526f91"), + BlockNumber: 8940380, + TxHash: common.HexToHash("0xfb37b7a88aa8ad14538d1e244a55939fa07c1828e5ca8168bf4edd56f5fc4d57"), TxIndex: 8, - BlockHash: common.HexToHash("0xe3dd2e05bd8b92833e20ed83e2171bbc06a9ec823232eca1730a807bd8f5edc0"), + BlockHash: common.HexToHash("0xf43ab2fd3cf0a7e08fcc16ec17bbc7f67417a37a4cd978d1d7ca32130c7f64be"), Index: 5, Removed: false, } var rawVatFoldLog, _ = json.Marshal(EthVatFoldLog) var VatFoldModel = vat_fold.VatFoldModel{ - Ilk: string(bytes.Trim(EthVatFoldLog.Topics[1].Bytes(), "\x00")), - Urn: string(bytes.Trim(EthVatFoldLog.Topics[2].Bytes(), "\x00")), - Rate: string(bytes.Trim(EthVatFoldLog.Topics[3].Bytes(), "\x00")), + Ilk: "REP", + Urn: "0x3728e9777B2a0a611ee0F89e00E01044ce4736d1", + Rate: "2", TransactionIndex: EthVatFoldLog.TxIndex, Raw: rawVatFoldLog, } diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go index 6270c647..c8bf0e1d 100644 --- a/pkg/transformers/vat_fold/converter.go +++ b/pkg/transformers/vat_fold/converter.go @@ -18,8 +18,11 @@ import ( "bytes" "encoding/json" "errors" + "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) type Converter interface { @@ -35,8 +38,8 @@ func (VatFoldConverter) ToModel(ethLog types.Log) (VatFoldModel, error) { } ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) - urn := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) - rate := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) + urn := common.HexToAddress(ethLog.Topics[2].String()).String() + rate := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() raw, err := json.Marshal(ethLog) return VatFoldModel{ diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go new file mode 100644 index 00000000..19fac0a1 --- /dev/null +++ b/pkg/transformers/vat_fold/repository.go @@ -0,0 +1,60 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerID int64, model VatFoldModel) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type VatFoldRepository struct { + db *postgres.DB +} + +func NewVatFoldRepository(db *postgres.DB) VatFoldRepository { + return VatFoldRepository{ + db: db, + } +} + +func (repository VatFoldRepository) Create(headerID int64, model VatFoldModel) error { + _, err := repository.db.Exec(`INSERT INTO maker.vat_fold (header_id, ilk, urn, rate, raw_log, tx_idx) + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + headerID, model.Ilk, model.Urn, model.Rate, model.Raw, model.TransactionIndex) + return err +} + +func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN maker.vat_fold on headers.id = header_id + WHERE header_id ISNULL + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + + return result, err +} diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go new file mode 100644 index 00000000..d703bc1e --- /dev/null +++ b/pkg/transformers/vat_fold/repository_test.go @@ -0,0 +1,145 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold_test + +import ( + "database/sql" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("", func() { + Describe("Create", func() { + It("adds a vat event", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatFoldRepository := vat_fold.NewVatFoldRepository(db) + + err = vatFoldRepository.Create(headerID, test_data.VatFoldModel) + + Expect(err).NotTo(HaveOccurred()) + var dbVatFold vat_fold.VatFoldModel + err = db.Get(&dbVatFold, `SELECT ilk, urn, rate, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbVatFold.Ilk).To(Equal(test_data.VatFoldModel.Ilk)) + Expect(dbVatFold.Urn).To(Equal(test_data.VatFoldModel.Urn)) + Expect(dbVatFold.Rate).To(Equal(test_data.VatFoldModel.Rate)) + Expect(dbVatFold.TransactionIndex).To(Equal(test_data.VatFoldModel.TransactionIndex)) + Expect(dbVatFold.Raw).To(MatchJSON(test_data.VatFoldModel.Raw)) + }) + + It("does not duplicate vat events", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatfoldRepository := vat_fold.NewVatFoldRepository(db) + err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + + err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("removes vat if corresponding header is deleted", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + vatfoldRepository := vat_fold.NewVatFoldRepository(db) + err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbVatFold vat_fold.VatFoldModel + err = db.Get(&dbVatFold, `SELECT ilk, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(sql.ErrNoRows)) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers with no associated vat event", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatfoldBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatfoldBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatfoldRepository := vat_fold.NewVatFoldRepository(db) + err := vatfoldRepository.Create(headerIDs[1], test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatfoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + vatFoldRepository := vat_fold.NewVatFoldRepository(db) + vatFoldRepositoryTwo := vat_fold.NewVatFoldRepository(dbTwo) + err := vatFoldRepository.Create(headerIDs[0], test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := vatFoldRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := vatFoldRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) From 46bba309380def434d571c8bed0ebfa8ce72972d Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 3 Oct 2018 19:28:33 +0200 Subject: [PATCH 10/30] Vat.fold: refactor repository tests --- pkg/transformers/vat_fold/repository_test.go | 157 +++++++++++-------- 1 file changed, 92 insertions(+), 65 deletions(-) diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index d703bc1e..6de7b3f4 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" @@ -28,21 +29,32 @@ import ( ) var _ = Describe("", func() { + Describe("Create", func() { - It("adds a vat event", func() { - db := test_config.NewTestDB(core.Node{}) + + var db *postgres.DB + var headerID int64 + var vatFoldRepository vat_fold.VatFoldRepository + + BeforeEach(func() { + db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + id, err := headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - vatFoldRepository := vat_fold.NewVatFoldRepository(db) + headerID = id + vatFoldRepository = vat_fold.NewVatFoldRepository(db) err = vatFoldRepository.Create(headerID, test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + }) - Expect(err).NotTo(HaveOccurred()) + It("adds a vat event", func() { var dbVatFold vat_fold.VatFoldModel - err = db.Get(&dbVatFold, `SELECT ilk, urn, rate, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) + err := db.Get(&dbVatFold, `SELECT ilk, urn, rate, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) + Expect(dbVatFold.Ilk).To(Equal(test_data.VatFoldModel.Ilk)) Expect(dbVatFold.Urn).To(Equal(test_data.VatFoldModel.Urn)) Expect(dbVatFold.Rate).To(Equal(test_data.VatFoldModel.Rate)) @@ -51,34 +63,16 @@ var _ = Describe("", func() { }) It("does not duplicate vat events", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - vatfoldRepository := vat_fold.NewVatFoldRepository(db) - err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) - Expect(err).NotTo(HaveOccurred()) - - err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) + err := vatFoldRepository.Create(headerID, test_data.VatFoldModel) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("removes vat if corresponding header is deleted", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - vatfoldRepository := vat_fold.NewVatFoldRepository(db) - err = vatfoldRepository.Create(headerID, test_data.VatFoldModel) + _, err := db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) - _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerID) - - Expect(err).NotTo(HaveOccurred()) var dbVatFold vat_fold.VatFoldModel err = db.Get(&dbVatFold, `SELECT ilk, tx_idx, raw_log FROM maker.vat_fold WHERE header_id = $1`, headerID) Expect(err).To(HaveOccurred()) @@ -87,59 +81,92 @@ var _ = Describe("", func() { }) Describe("MissingHeaders", func() { - It("returns headers with no associated vat event", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - startingBlockNumber := int64(1) - vatfoldBlockNumber := int64(2) - endingBlockNumber := int64(3) - blockNumbers := []int64{startingBlockNumber, vatfoldBlockNumber, endingBlockNumber, endingBlockNumber + 1} - var headerIDs []int64 - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - vatfoldRepository := vat_fold.NewVatFoldRepository(db) - err := vatfoldRepository.Create(headerIDs[1], test_data.VatFoldModel) - Expect(err).NotTo(HaveOccurred()) - headers, err := vatfoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + It("returns headers with no associated vat event", func() { + startBlock := int64(1) + eventBlock := int64(2) + finalBlock := int64(3) + blockNumbers := []int64{startBlock, eventBlock, finalBlock, finalBlock + 1} + + repository := initRepository( + repositoryOptions{ + cleanDB: true, + blockNumbers: blockNumbers, + storeEvent: true, + storedEventBlockNumber: 1, + }, + ) + + headers, err := repository.MissingHeaders(startBlock, finalBlock) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(2)) - Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) - Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[0].BlockNumber).To(Or(Equal(startBlock), Equal(finalBlock))) + Expect(headers[1].BlockNumber).To(Or(Equal(startBlock), Equal(finalBlock))) }) It("only returns headers associated with the current node", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) blockNumbers := []int64{1, 2, 3} - headerRepository := repositories.NewHeaderRepository(db) - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - var headerIDs []int64 - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) - Expect(err).NotTo(HaveOccurred()) - headerIDs = append(headerIDs, headerID) - _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) - Expect(err).NotTo(HaveOccurred()) - } - vatFoldRepository := vat_fold.NewVatFoldRepository(db) - vatFoldRepositoryTwo := vat_fold.NewVatFoldRepository(dbTwo) - err := vatFoldRepository.Create(headerIDs[0], test_data.VatFoldModel) - Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := vatFoldRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + repositoryOne := initRepository( + repositoryOptions{ + blockNumbers: blockNumbers, + nodeID: "first", + cleanDB: true, + storeEvent: true, + }, + ) + + repositoryTwo := initRepository( + repositoryOptions{ + blockNumbers: blockNumbers, + nodeID: "second", + cleanDB: false, + storeEvent: false, + }, + ) + + nodeOneMissingHeaders, err := repositoryOne.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) - nodeTwoMissingHeaders, err := vatFoldRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) }) + +// ------------------------------------------------------------------------------------------- + +type repositoryOptions struct { + blockNumbers []int64 + nodeID string + cleanDB bool + storeEvent bool + storedEventBlockNumber int64 +} + +func initRepository(options repositoryOptions) vat_fold.VatFoldRepository { + db := test_config.NewTestDB(core.Node{ID: options.nodeID}) + if options.cleanDB { + test_config.CleanTestDB(db) + } + + headerRepository := repositories.NewHeaderRepository(db) + vatfoldRepository := vat_fold.NewVatFoldRepository(db) + + var headerIDs []int64 + for _, n := range options.blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + + if options.storeEvent { + err := vatfoldRepository.Create(headerIDs[options.storedEventBlockNumber], test_data.VatFoldModel) + Expect(err).NotTo(HaveOccurred()) + } + + return vatfoldRepository +} From c8cb079da58065a96503aa8f8c99cc6974481a33 Mon Sep 17 00:00:00 2001 From: David Terry Date: Thu, 4 Oct 2018 14:34:30 +0300 Subject: [PATCH 11/30] Vat.fold: add transformer & mocks --- .../test_data/mocks/vat_fold/converter.go | 36 ++++ .../test_data/mocks/vat_fold/repository.go | 54 ++++++ pkg/transformers/vat_fold/config.go | 27 +++ pkg/transformers/vat_fold/transformer.go | 74 ++++++++ pkg/transformers/vat_fold/transformer_test.go | 179 ++++++++++++++++++ 5 files changed, 370 insertions(+) create mode 100644 pkg/transformers/test_data/mocks/vat_fold/converter.go create mode 100644 pkg/transformers/test_data/mocks/vat_fold/repository.go create mode 100644 pkg/transformers/vat_fold/config.go create mode 100644 pkg/transformers/vat_fold/transformer.go create mode 100644 pkg/transformers/vat_fold/transformer_test.go diff --git a/pkg/transformers/test_data/mocks/vat_fold/converter.go b/pkg/transformers/test_data/mocks/vat_fold/converter.go new file mode 100644 index 00000000..9625cf78 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_fold/converter.go @@ -0,0 +1,36 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" +) + +type MockVatFoldConverter struct { + converterErr error + PassedLog types.Log +} + +func (converter *MockVatFoldConverter) ToModel(ethLog types.Log) (vat_fold.VatFoldModel, error) { + converter.PassedLog = ethLog + return test_data.VatFoldModel, converter.converterErr +} + +func (converter *MockVatFoldConverter) SetConverterError(e error) { + converter.converterErr = e +} diff --git a/pkg/transformers/test_data/mocks/vat_fold/repository.go b/pkg/transformers/test_data/mocks/vat_fold/repository.go new file mode 100644 index 00000000..79e973d0 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_fold/repository.go @@ -0,0 +1,54 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" +) + +type MockVatFoldRepository struct { + createErr error + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModel vat_fold.VatFoldModel +} + +func (repository *MockVatFoldRepository) Create(headerID int64, model vat_fold.VatFoldModel) error { + repository.PassedHeaderID = headerID + repository.PassedModel = model + return repository.createErr +} + +func (repository *MockVatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatFoldRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatFoldRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatFoldRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/vat_fold/config.go b/pkg/transformers/vat_fold/config.go new file mode 100644 index 00000000..56e8983d --- /dev/null +++ b/pkg/transformers/vat_fold/config.go @@ -0,0 +1,27 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var VatFoldConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatFoldSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_fold/transformer.go b/pkg/transformers/vat_fold/transformer.go new file mode 100644 index 00000000..78237fb0 --- /dev/null +++ b/pkg/transformers/vat_fold/transformer.go @@ -0,0 +1,74 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold + +import ( + "log" + + "github.com/ethereum/go-ethereum/common" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type VatFoldTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatFoldTransformerInitializer) NewVatFoldTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatFoldConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatFoldRepository(db) + return VatFoldTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatFoldTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatFoldTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat fold event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatFoldSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatFoldConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + for _, log := range matchingLogs { + model, err := transformer.Converter.ToModel(log) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, model) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/transformers/vat_fold/transformer_test.go b/pkg/transformers/vat_fold/transformer_test.go new file mode 100644 index 00000000..57370d2b --- /dev/null +++ b/pkg/transformers/vat_fold/transformer_test.go @@ -0,0 +1,179 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_fold_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_fold_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_fold" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" +) + +type setupOptions struct { + setMissingHeadersError bool + setFetcherError bool + setConverterError bool + setCreateError bool + fetchedLogs []types.Log + missingHeaders []core.Header +} + +func setup(options setupOptions) ( + vat_fold.VatFoldTransformer, + *mocks.MockLogFetcher, + *vat_fold_mocks.MockVatFoldConverter, + *vat_fold_mocks.MockVatFoldRepository, +) { + fetcher := &mocks.MockLogFetcher{} + if options.setFetcherError { + fetcher.SetFetcherError(fakes.FakeError) + } + if len(options.fetchedLogs) > 0 { + fetcher.SetFetchedLogs(options.fetchedLogs) + } + + converter := &vat_fold_mocks.MockVatFoldConverter{} + if options.setConverterError { + converter.SetConverterError(fakes.FakeError) + } + + repository := &vat_fold_mocks.MockVatFoldRepository{} + if options.setMissingHeadersError { + repository.SetMissingHeadersErr(fakes.FakeError) + } + if options.setCreateError { + repository.SetCreateError(fakes.FakeError) + } + if len(options.missingHeaders) > 0 { + repository.SetMissingHeaders(options.missingHeaders) + } + + transformer := vat_fold.VatFoldTransformer{ + Config: vat_fold.VatFoldConfig, + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + return transformer, fetcher, converter, repository +} + +var _ = Describe("Vat fold transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + transformer, _, _, repository := setup(setupOptions{}) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_fold.VatFoldConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_fold.VatFoldConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + transformer, _, _, _ := setup(setupOptions{ + setMissingHeadersError: true, + }) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + transformer, fetcher, _, _ := setup(setupOptions{ + missingHeaders: []core.Header{{BlockNumber: 1}, {BlockNumber: 2}}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_fold.VatFoldConfig.ContractAddresses, vat_fold.VatFoldConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatFoldSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + transformer, _, _, _ := setup(setupOptions{ + setFetcherError: true, + missingHeaders: []core.Header{{BlockNumber: 1}}, + }) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + transformer, _, converter, _ := setup(setupOptions{ + fetchedLogs: []types.Log{test_data.EthVatFoldLog}, + missingHeaders: []core.Header{{BlockNumber: 1}}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLog).To(Equal(test_data.EthVatFoldLog)) + }) + + It("returns error if converter returns error", func() { + transformer, _, _, _ := setup(setupOptions{ + setConverterError: true, + fetchedLogs: []types.Log{test_data.EthVatFoldLog}, + missingHeaders: []core.Header{{BlockNumber: 1}}, + }) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat fold model", func() { + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + transformer, _, _, repository := setup(setupOptions{ + fetchedLogs: []types.Log{test_data.EthVatFoldLog}, + missingHeaders: []core.Header{fakeHeader}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModel).To(Equal(test_data.VatFoldModel)) + }) + + It("returns error if repository returns error for create", func() { + transformer, _, _, _ := setup(setupOptions{ + fetchedLogs: []types.Log{test_data.EthVatFoldLog}, + missingHeaders: []core.Header{{BlockNumber: 1, Id: 2}}, + setCreateError: true, + }) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) From dde57ea70ba01fb7c7d2dc58f74eecc4222eef17 Mon Sep 17 00:00:00 2001 From: David Terry Date: Thu, 4 Oct 2018 14:36:52 +0300 Subject: [PATCH 12/30] Vat.fold: run transformer --- pkg/transformers/transformers.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 6aa22de7..8ea44502 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -37,6 +37,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" ) @@ -63,6 +64,7 @@ var ( PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer + VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatInitConfig}.NewVatFoldTransformer ) func TransformerInitializers() []shared.TransformerInitializer { @@ -86,5 +88,6 @@ func TransformerInitializers() []shared.TransformerInitializer { PriceFeedTransformerInitializer, TendTransformerInitializer, VatInitTransformerInitializer, + VatFoldTransformerInitializer, } } From 53a74c39f7a6d531a3cc39768833818a976e19bd Mon Sep 17 00:00:00 2001 From: David Terry Date: Thu, 4 Oct 2018 17:47:03 +0300 Subject: [PATCH 13/30] Vat.fold: correct typo --- pkg/transformers/transformers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 8ea44502..38327842 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -64,7 +64,7 @@ var ( PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer - VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatInitConfig}.NewVatFoldTransformer + VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer ) func TransformerInitializers() []shared.TransformerInitializer { From 8fb573351ec0aea5bc44d7252063c2e68053538a Mon Sep 17 00:00:00 2001 From: David Terry Date: Thu, 4 Oct 2018 17:47:13 +0300 Subject: [PATCH 14/30] Vat.fold: add to continuousLogSync --- cmd/continuousLogSync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 7d9d0e5b..60c5f6b3 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -100,6 +100,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap["priceFeed"] = transformers.PriceFeedTransformerInitializer transformerInitializerMap["tend"] = transformers.TendTransformerInitializer transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer + transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer return transformerInitializerMap } From f79a017b488e46d511e1b470578cce1085b01dd1 Mon Sep 17 00:00:00 2001 From: David Terry Date: Thu, 4 Oct 2018 18:48:06 +0300 Subject: [PATCH 15/30] Vat.fold: store ilk and urn as text instead of bytea --- db/migrations/1538510582_create_vat_fold_table.up.sql | 4 ++-- db/schema.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/migrations/1538510582_create_vat_fold_table.up.sql b/db/migrations/1538510582_create_vat_fold_table.up.sql index 327719e1..ad5ca113 100644 --- a/db/migrations/1538510582_create_vat_fold_table.up.sql +++ b/db/migrations/1538510582_create_vat_fold_table.up.sql @@ -1,8 +1,8 @@ CREATE TABLE maker.vat_fold ( id SERIAL PRIMARY KEY, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, - ilk bytea, - urn bytea, + ilk text, + urn text, rate numeric, tx_idx INTEGER NOT NULL, raw_log JSONB, diff --git a/db/schema.sql b/db/schema.sql index 83e79100..7fa2d53e 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -704,8 +704,8 @@ ALTER SEQUENCE maker.tend_id_seq OWNED BY maker.tend.id; CREATE TABLE maker.vat_fold ( id integer NOT NULL, header_id integer NOT NULL, - ilk bytea, - urn bytea, + ilk text, + urn text, rate numeric, tx_idx integer NOT NULL, raw_log jsonb From 3e9901864b0b5a3c45c82736c003b74404bd0c39 Mon Sep 17 00:00:00 2001 From: David Terry Date: Tue, 9 Oct 2018 13:34:22 +0300 Subject: [PATCH 16/30] vat.fold: switch to batched interface --- .../test_data/mocks/vat_fold/converter.go | 8 ++-- .../test_data/mocks/vat_fold/repository.go | 35 ++++++++++++---- pkg/transformers/vat_fold/converter.go | 42 +++++++++++-------- pkg/transformers/vat_fold/converter_test.go | 6 +-- pkg/transformers/vat_fold/repository.go | 37 ++++++++++++---- pkg/transformers/vat_fold/repository_test.go | 9 ++-- pkg/transformers/vat_fold/transformer.go | 16 ++++--- pkg/transformers/vat_fold/transformer_test.go | 4 +- 8 files changed, 104 insertions(+), 53 deletions(-) diff --git a/pkg/transformers/test_data/mocks/vat_fold/converter.go b/pkg/transformers/test_data/mocks/vat_fold/converter.go index 9625cf78..c9b992bf 100644 --- a/pkg/transformers/test_data/mocks/vat_fold/converter.go +++ b/pkg/transformers/test_data/mocks/vat_fold/converter.go @@ -23,12 +23,12 @@ import ( type MockVatFoldConverter struct { converterErr error - PassedLog types.Log + PassedLogs []types.Log } -func (converter *MockVatFoldConverter) ToModel(ethLog types.Log) (vat_fold.VatFoldModel, error) { - converter.PassedLog = ethLog - return test_data.VatFoldModel, converter.converterErr +func (converter *MockVatFoldConverter) ToModels(ethLogs []types.Log) ([]vat_fold.VatFoldModel, error) { + converter.PassedLogs = ethLogs + return []vat_fold.VatFoldModel{test_data.VatFoldModel}, converter.converterErr } func (converter *MockVatFoldConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/vat_fold/repository.go b/pkg/transformers/test_data/mocks/vat_fold/repository.go index 79e973d0..55deb829 100644 --- a/pkg/transformers/test_data/mocks/vat_fold/repository.go +++ b/pkg/transformers/test_data/mocks/vat_fold/repository.go @@ -15,32 +15,45 @@ package vat_fold import ( + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" ) type MockVatFoldRepository struct { - createErr error - missingHeaders []core.Header - missingHeadersErr error - PassedStartingBlockNumber int64 - PassedEndingBlockNumber int64 - PassedHeaderID int64 - PassedModel vat_fold.VatFoldModel + createErr error + markHeaderCheckedErr error + markHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_fold.VatFoldModel } -func (repository *MockVatFoldRepository) Create(headerID int64, model vat_fold.VatFoldModel) error { +func (repository *MockVatFoldRepository) Create(headerID int64, models []vat_fold.VatFoldModel) error { repository.PassedHeaderID = headerID - repository.PassedModel = model + repository.PassedModels = models return repository.createErr } +func (repository *MockVatFoldRepository) MarkHeaderChecked(headerID int64) error { + repository.markHeaderCheckedPassedHeaderID = headerID + return repository.markHeaderCheckedErr +} + func (repository *MockVatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber return repository.missingHeaders, repository.missingHeadersErr } +func (repository *MockVatFoldRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + func (repository *MockVatFoldRepository) SetMissingHeadersErr(e error) { repository.missingHeadersErr = e } @@ -52,3 +65,7 @@ func (repository *MockVatFoldRepository) SetMissingHeaders(headers []core.Header func (repository *MockVatFoldRepository) SetCreateError(e error) { repository.createErr = e } + +func (repository *MockVatFoldRepository) AssertMarkHeaderCheckedCalledWith(i int64) { + Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) +} diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go index c8bf0e1d..4f57919c 100644 --- a/pkg/transformers/vat_fold/converter.go +++ b/pkg/transformers/vat_fold/converter.go @@ -26,29 +26,35 @@ import ( ) type Converter interface { - ToModel(ethLog types.Log) (VatFoldModel, error) + ToModels(ethLogs []types.Log) ([]VatFoldModel, error) } type VatFoldConverter struct{} -func (VatFoldConverter) ToModel(ethLog types.Log) (VatFoldModel, error) { - err := verifyLog(ethLog) - if err != nil { - return VatFoldModel{}, err +func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]VatFoldModel, error) { + var models []VatFoldModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + + ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) + urn := common.BytesToAddress(ethLog.Topics[2].Bytes()).String() + rate := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() + raw, err := json.Marshal(ethLog) + + model := VatFoldModel{ + Ilk: ilk, + Urn: urn, + Rate: rate, + TransactionIndex: ethLog.TxIndex, + Raw: raw, + } + + models = append(models, model) } - - ilk := string(bytes.Trim(ethLog.Topics[1].Bytes(), "\x00")) - urn := common.HexToAddress(ethLog.Topics[2].String()).String() - rate := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() - raw, err := json.Marshal(ethLog) - - return VatFoldModel{ - Ilk: ilk, - Urn: urn, - Rate: rate, - TransactionIndex: ethLog.TxIndex, - Raw: raw, - }, err + return models, nil } func verifyLog(log types.Log) error { diff --git a/pkg/transformers/vat_fold/converter_test.go b/pkg/transformers/vat_fold/converter_test.go index 021096db..97d687ad 100644 --- a/pkg/transformers/vat_fold/converter_test.go +++ b/pkg/transformers/vat_fold/converter_test.go @@ -28,7 +28,7 @@ var _ = Describe("Vat fold converter", func() { converter := vat_fold.VatFoldConverter{} badLog := types.Log{} - _, err := converter.ToModel(badLog) + _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) @@ -36,9 +36,9 @@ var _ = Describe("Vat fold converter", func() { It("converts a log to an model", func() { converter := vat_fold.VatFoldConverter{} - model, err := converter.ToModel(test_data.EthVatFoldLog) + model, err := converter.ToModels([]types.Log{test_data.EthVatFoldLog}) Expect(err).NotTo(HaveOccurred()) - Expect(model).To(Equal(test_data.VatFoldModel)) + Expect(model).To(Equal([]vat_fold.VatFoldModel{test_data.VatFoldModel})) }) }) diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 19fac0a1..d68b93a8 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -20,7 +20,8 @@ import ( ) type Repository interface { - Create(headerID int64, model VatFoldModel) error + Create(headerID int64, models []VatFoldModel) error + MarkHeaderChecked(headerID int64) error MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) } @@ -34,10 +35,31 @@ func NewVatFoldRepository(db *postgres.DB) VatFoldRepository { } } -func (repository VatFoldRepository) Create(headerID int64, model VatFoldModel) error { - _, err := repository.db.Exec(`INSERT INTO maker.vat_fold (header_id, ilk, urn, rate, raw_log, tx_idx) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, - headerID, model.Ilk, model.Urn, model.Rate, model.Raw, model.TransactionIndex) +func (repository VatFoldRepository) Create(headerID int64, models []VatFoldModel) error { + tx, err := repository.db.Begin() + if err != nil { + return err + } + for _, model := range models { + _, err = tx.Exec( + `INSERT into maker.vat_fold (header_id, ilk, urn, rate, tx_idx, raw_log) + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + headerID, model.Ilk, model.Urn, model.Rate, model.TransactionIndex, model.Raw, + ) + if err != nil { + tx.Rollback() + return err + } + } + + return tx.Commit() +} + +func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_fold_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_fold_checked = $2`, headerID, true) return err } @@ -46,8 +68,8 @@ func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBl err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN maker.vat_fold on headers.id = header_id - WHERE header_id ISNULL + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_fold_checked IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3`, @@ -55,6 +77,5 @@ func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBl endingBlockNumber, repository.db.Node.ID, ) - return result, err } diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index 6de7b3f4..a8c17c16 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -46,7 +46,7 @@ var _ = Describe("", func() { headerID = id vatFoldRepository = vat_fold.NewVatFoldRepository(db) - err = vatFoldRepository.Create(headerID, test_data.VatFoldModel) + err = vatFoldRepository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -63,7 +63,7 @@ var _ = Describe("", func() { }) It("does not duplicate vat events", func() { - err := vatFoldRepository.Create(headerID, test_data.VatFoldModel) + err := vatFoldRepository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -164,7 +164,10 @@ func initRepository(options repositoryOptions) vat_fold.VatFoldRepository { } if options.storeEvent { - err := vatfoldRepository.Create(headerIDs[options.storedEventBlockNumber], test_data.VatFoldModel) + err := vatfoldRepository.Create( + headerIDs[options.storedEventBlockNumber], + []vat_fold.VatFoldModel{test_data.VatFoldModel}, + ) Expect(err).NotTo(HaveOccurred()) } diff --git a/pkg/transformers/vat_fold/transformer.go b/pkg/transformers/vat_fold/transformer.go index 78237fb0..17c6a321 100644 --- a/pkg/transformers/vat_fold/transformer.go +++ b/pkg/transformers/vat_fold/transformer.go @@ -59,16 +59,20 @@ func (transformer VatFoldTransformer) Execute() error { if err != nil { return err } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { return err } } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } } return nil } diff --git a/pkg/transformers/vat_fold/transformer_test.go b/pkg/transformers/vat_fold/transformer_test.go index 57370d2b..f263bc90 100644 --- a/pkg/transformers/vat_fold/transformer_test.go +++ b/pkg/transformers/vat_fold/transformer_test.go @@ -134,7 +134,7 @@ var _ = Describe("Vat fold transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedLog).To(Equal(test_data.EthVatFoldLog)) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.EthVatFoldLog})) }) It("returns error if converter returns error", func() { @@ -161,7 +161,7 @@ var _ = Describe("Vat fold transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModel).To(Equal(test_data.VatFoldModel)) + Expect(repository.PassedModels).To(Equal([]vat_fold.VatFoldModel{test_data.VatFoldModel})) }) It("returns error if repository returns error for create", func() { From 5cc4748f1081ecb307c4261bf6b9cd957c82e2e3 Mon Sep 17 00:00:00 2001 From: David Terry Date: Tue, 9 Oct 2018 13:34:55 +0300 Subject: [PATCH 17/30] vat.fold: clean up table in test db --- test_config/test_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test_config/test_config.go b/test_config/test_config.go index 12717fe9..8b6d37f7 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -90,6 +90,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.price_feeds") db.MustExec("DELETE FROM maker.tend") db.MustExec("DELETE FROM maker.vat_init") + db.MustExec("DELETE FROM maker.vat_fold") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts") From e040383fa5f7602c2bd2390b01cbbc213968dc66 Mon Sep 17 00:00:00 2001 From: David Terry Date: Tue, 9 Oct 2018 14:10:42 +0300 Subject: [PATCH 18/30] vat.fold: add test for signature generation --- pkg/transformers/shared/event_signature_generator_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index 3bed6e80..e19cdc3e 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -119,6 +119,13 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + It("gets the vat init method signature", func() { + expected := "fold(bytes32,bytes32,int256)" + actual := shared.GetSolidityMethodSignature(shared.VatABI, "fold") + + Expect(expected).To(Equal(actual)) + }) + It("gets the flip deal method signature", func() { expected := "deal(uint256)" actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "deal") From 81dec589fc2c1d28f34eeaa79636e183db9afa70 Mon Sep 17 00:00:00 2001 From: David Terry Date: Tue, 9 Oct 2018 14:11:02 +0300 Subject: [PATCH 19/30] vat.fold: rollback broken test refactoring --- pkg/transformers/vat_fold/repository_test.go | 140 +++++++++---------- 1 file changed, 66 insertions(+), 74 deletions(-) diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index a8c17c16..68360b54 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -82,94 +82,86 @@ var _ = Describe("", func() { Describe("MissingHeaders", func() { - It("returns headers with no associated vat event", func() { - startBlock := int64(1) - eventBlock := int64(2) - finalBlock := int64(3) - blockNumbers := []int64{startBlock, eventBlock, finalBlock, finalBlock + 1} + It("returns headers that haven't been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatFoldRepository := vat_fold.NewVatFoldRepository(db) + err := vatFoldRepository.MarkHeaderChecked(headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) - repository := initRepository( - repositoryOptions{ - cleanDB: true, - blockNumbers: blockNumbers, - storeEvent: true, - storedEventBlockNumber: 1, - }, - ) - - headers, err := repository.MissingHeaders(startBlock, finalBlock) + headers, err := vatFoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(2)) - Expect(headers[0].BlockNumber).To(Or(Equal(startBlock), Equal(finalBlock))) - Expect(headers[1].BlockNumber).To(Or(Equal(startBlock), Equal(finalBlock))) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only treats headers as checked if vat fold logs have been checked", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + vatGrabdBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + vatFoldRepository := vat_fold.NewVatFoldRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := vatFoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(vatGrabdBlockNumber))) }) It("only returns headers associated with the current node", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + vatFoldRepository := vat_fold.NewVatFoldRepository(db) + vatFoldRepositoryTwo := vat_fold.NewVatFoldRepository(dbTwo) + err := vatFoldRepository.MarkHeaderChecked(headerIDs[0]) + Expect(err).NotTo(HaveOccurred()) - repositoryOne := initRepository( - repositoryOptions{ - blockNumbers: blockNumbers, - nodeID: "first", - cleanDB: true, - storeEvent: true, - }, - ) - - repositoryTwo := initRepository( - repositoryOptions{ - blockNumbers: blockNumbers, - nodeID: "second", - cleanDB: false, - storeEvent: false, - }, - ) - - nodeOneMissingHeaders, err := repositoryOne.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeOneMissingHeaders, err := vatFoldRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) - nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeTwoMissingHeaders, err := vatFoldRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) }) - -// ------------------------------------------------------------------------------------------- - -type repositoryOptions struct { - blockNumbers []int64 - nodeID string - cleanDB bool - storeEvent bool - storedEventBlockNumber int64 -} - -func initRepository(options repositoryOptions) vat_fold.VatFoldRepository { - db := test_config.NewTestDB(core.Node{ID: options.nodeID}) - if options.cleanDB { - test_config.CleanTestDB(db) - } - - headerRepository := repositories.NewHeaderRepository(db) - vatfoldRepository := vat_fold.NewVatFoldRepository(db) - - var headerIDs []int64 - for _, n := range options.blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - - if options.storeEvent { - err := vatfoldRepository.Create( - headerIDs[options.storedEventBlockNumber], - []vat_fold.VatFoldModel{test_data.VatFoldModel}, - ) - Expect(err).NotTo(HaveOccurred()) - } - - return vatfoldRepository -} From ecef1cbcdbb0c25100d6b0170155753cf4ea8702 Mon Sep 17 00:00:00 2001 From: David Terry Date: Tue, 9 Oct 2018 14:25:27 +0300 Subject: [PATCH 20/30] transformers: add step by step docs --- pkg/transformers/DOCUMENTATION.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index c53130ad..286983cb 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -23,6 +23,22 @@ The transformer process for each of these different log types is the same, excep ## Creating a Transformer +1. Pull an example event (from kovan / ganache etc.) +1. Add event sig to [`constants.go`](./shared/constants.go) +1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go) +1. Create DB table (using [`create_migration`](../../scripts/create_migration)) +1. Create columns in `checked_headers` in the _same_ migration +1. Add a line to clean the new table `CleanTestDB` (in [`test_config.go`](../../test_config/test_config.go)) +1. Define `model.go` +1. Create test event in [`test_data`](./test_data) +1. Write converter + converter tests +1. Write repository + repository tests +1. Create converter + repository mocks +1. Create transformer + transformer tests +1. Wire up transformer in [`transformers.go`](./transformers.go) +1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go) +1. Manually trigger an event and check that it gets persisted to postgres + **Fetching Logs** 1. Generate an example raw log event, by either: From f05f2e3ae955a5a88cbfaf37e24758640680457e Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:37:31 +0300 Subject: [PATCH 21/30] vat.fold: correct table name in down migration --- db/migrations/1538510582_create_vat_fold_table.down.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrations/1538510582_create_vat_fold_table.down.sql b/db/migrations/1538510582_create_vat_fold_table.down.sql index bfbedda7..d63b6597 100644 --- a/db/migrations/1538510582_create_vat_fold_table.down.sql +++ b/db/migrations/1538510582_create_vat_fold_table.down.sql @@ -1,3 +1,3 @@ -DROP TABLE maker.flop_kick; +DROP TABLE maker.vat_fold; ALTER TABLE public.checked_headers - DROP COLUMN flop_kick_checked; + DROP COLUMN vat_fold_checked; From 724839ce4a7517a30f8a918860d6be1d7939ab46 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:37:59 +0300 Subject: [PATCH 22/30] documentation: document vat event types --- pkg/transformers/DOCUMENTATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index 286983cb..b9298baa 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -17,7 +17,8 @@ For Maker, vulcanize will be run in `lightSync` mode, so it will store all heade For Maker there are two main types of log events that we're tracking: 1. Custom events that are defined in the contract solidity code. -1. LogNote events which utilize the [DSNote library](https://github.com/dapphub/ds-note). +1. `LogNote` events which utilize the [DSNote library](https://github.com/dapphub/ds-note). +1. `Note` events in the `Vat` The transformer process for each of these different log types is the same, except for the converting process, as denoted below. From dde57178f171a58e6d6fde0f0113ca1e22d825ab Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:38:57 +0300 Subject: [PATCH 23/30] vat.fold: break out of loop in transformer if no logs found --- pkg/transformers/vat_fold/transformer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/transformers/vat_fold/transformer.go b/pkg/transformers/vat_fold/transformer.go index 17c6a321..7f0cd298 100644 --- a/pkg/transformers/vat_fold/transformer.go +++ b/pkg/transformers/vat_fold/transformer.go @@ -64,6 +64,7 @@ func (transformer VatFoldTransformer) Execute() error { if err != nil { return err } + continue } models, err := transformer.Converter.ToModels(matchingLogs) if err != nil { From 2e89df081cb408665531d3ca75a6eaea9debe20e Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:39:20 +0300 Subject: [PATCH 24/30] test_config: clean checked_headers table --- test_config/test_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test_config/test_config.go b/test_config/test_config.go index 8b6d37f7..a225a4bf 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -75,6 +75,7 @@ func NewTestDB(node core.Node) *postgres.DB { func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM blocks") db.MustExec("DELETE FROM headers") + db.MustExec("DELETE FROM checked_headers") db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") db.MustExec("DELETE FROM maker.bite") From 7e7ad13de62da088e780822a4fd88382c398b05b Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:39:33 +0300 Subject: [PATCH 25/30] vat.fold: handle errors --- pkg/transformers/vat_fold/converter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/transformers/vat_fold/converter.go b/pkg/transformers/vat_fold/converter.go index 4f57919c..fc5e6150 100644 --- a/pkg/transformers/vat_fold/converter.go +++ b/pkg/transformers/vat_fold/converter.go @@ -44,6 +44,10 @@ func (VatFoldConverter) ToModels(ethLogs []types.Log) ([]VatFoldModel, error) { rate := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() raw, err := json.Marshal(ethLog) + if err != nil { + return models, err + } + model := VatFoldModel{ Ilk: ilk, Urn: urn, From 3778d7ac06275ea5a2f60f45a346f6f510455754 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:40:25 +0300 Subject: [PATCH 26/30] vat.fold: add transformer tests for MarkHeaderChecked --- pkg/transformers/vat_fold/transformer_test.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pkg/transformers/vat_fold/transformer_test.go b/pkg/transformers/vat_fold/transformer_test.go index f263bc90..8d5c74a1 100644 --- a/pkg/transformers/vat_fold/transformer_test.go +++ b/pkg/transformers/vat_fold/transformer_test.go @@ -176,4 +176,40 @@ var _ = Describe("Vat fold transformer", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) + + It("marks the header as checked when there are no logs", func() { + header := core.Header{Id: GinkgoRandomSeed()} + transformer, _, _, repository := setup(setupOptions{ + missingHeaders: []core.Header{header}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(header.Id)) + }) + + It("doesn't call MarkHeaderChecked when there are logs", func() { + transformer, _, _, repository := setup(setupOptions{ + missingHeaders: []core.Header{{Id: GinkgoRandomSeed()}}, + fetchedLogs: []types.Log{test_data.EthVatFoldLog}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(int64(0))) + }) + + It("returns an error if MarkHeaderChecked fails", func() { + transformer, _, _, _ := setup(setupOptions{ + missingHeaders: []core.Header{{Id: GinkgoRandomSeed()}}, + setMissingHeadersError: true, + }) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) }) From 1273acb733ba1a2413084050c37bdc5bb1fbf11d Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 13:41:26 +0300 Subject: [PATCH 27/30] vat.fold: add repository tests for MarkHeaderChecked --- .../test_data/mocks/vat_fold/repository.go | 6 +-- pkg/transformers/vat_fold/repository.go | 30 +++++++---- pkg/transformers/vat_fold/repository_test.go | 54 ++++++++++++++++++- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/pkg/transformers/test_data/mocks/vat_fold/repository.go b/pkg/transformers/test_data/mocks/vat_fold/repository.go index 55deb829..bbe2fec2 100644 --- a/pkg/transformers/test_data/mocks/vat_fold/repository.go +++ b/pkg/transformers/test_data/mocks/vat_fold/repository.go @@ -24,7 +24,7 @@ import ( type MockVatFoldRepository struct { createErr error markHeaderCheckedErr error - markHeaderCheckedPassedHeaderID int64 + MarkHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header missingHeadersErr error PassedStartingBlockNumber int64 @@ -40,7 +40,7 @@ func (repository *MockVatFoldRepository) Create(headerID int64, models []vat_fol } func (repository *MockVatFoldRepository) MarkHeaderChecked(headerID int64) error { - repository.markHeaderCheckedPassedHeaderID = headerID + repository.MarkHeaderCheckedPassedHeaderID = headerID return repository.markHeaderCheckedErr } @@ -67,5 +67,5 @@ func (repository *MockVatFoldRepository) SetCreateError(e error) { } func (repository *MockVatFoldRepository) AssertMarkHeaderCheckedCalledWith(i int64) { - Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) + Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(i)) } diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index d68b93a8..6f81bce9 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -26,26 +26,33 @@ type Repository interface { } type VatFoldRepository struct { - db *postgres.DB + DB *postgres.DB } func NewVatFoldRepository(db *postgres.DB) VatFoldRepository { return VatFoldRepository{ - db: db, + DB: db, } } func (repository VatFoldRepository) Create(headerID int64, models []VatFoldModel) error { - tx, err := repository.db.Begin() + tx, err := repository.DB.Begin() if err != nil { return err } for _, model := range models { _, err = tx.Exec( `INSERT into maker.vat_fold (header_id, ilk, urn, rate, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, headerID, model.Ilk, model.Urn, model.Rate, model.TransactionIndex, model.Raw, ) + _, err = tx.Exec( + `INSERT INTO public.checked_headers (header_id, vat_fold_checked) + VALUES($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_fold_checked = $2`, + headerID, true, + ) if err != nil { tx.Rollback() return err @@ -56,16 +63,19 @@ func (repository VatFoldRepository) Create(headerID int64, models []VatFoldModel } func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_fold_checked) - VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET vat_fold_checked = $2`, headerID, true) + _, err := repository.DB.Exec( + `INSERT INTO public.checked_headers (header_id, vat_fold_checked) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_fold_checked = $2`, + headerID, true, + ) return err } func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -75,7 +85,7 @@ func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBl AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index 68360b54..38b28397 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -28,7 +28,7 @@ import ( "github.com/vulcanize/vulcanizedb/test_config" ) -var _ = Describe("", func() { +var _ = Describe("Vat.fold repository", func() { Describe("Create", func() { @@ -80,6 +80,58 @@ var _ = Describe("", func() { }) }) + Describe("MarkHeaderChecked", func() { + var db *postgres.DB + var headerID int64 + var repository vat_fold.VatFoldRepository + + type CheckedHeaderResult struct { + VatFoldChecked bool `db:"vat_fold_checked"` + } + + BeforeEach(func() { + node := test_config.NewTestNode() + + db = test_config.NewTestDB(node) + test_config.CleanTestDB(db) + + headerRepository := repositories.NewHeaderRepository(db) + id, err := headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + headerID = id + + repository = vat_fold.NewVatFoldRepository(db) + Expect(err).NotTo(HaveOccurred()) + }) + + It("creates a new checked header record", func() { + err := repository.MarkHeaderChecked(headerID) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult = CheckedHeaderResult{} + err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFoldChecked).To(BeTrue()) + }) + + It("updates an existing checked header", func() { + _, err := repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES($1)`, headerID) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult CheckedHeaderResult + err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFoldChecked).To(BeFalse()) + + err = repository.MarkHeaderChecked(headerID) + Expect(err).NotTo(HaveOccurred()) + + err = db.Get(&checkedHeaderResult, `SELECT vat_fold_checked FROM checked_headers WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFoldChecked).To(BeTrue()) + }) + }) + Describe("MissingHeaders", func() { It("returns headers that haven't been checked", func() { From 95f1c94bf5a271a99e4d76635cd387e9855917fb Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 14:03:37 +0300 Subject: [PATCH 28/30] vat.fold: check for errors in insert before updating checked_headers --- pkg/transformers/vat_fold/repository.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 6f81bce9..f89b8057 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -46,6 +46,10 @@ func (repository VatFoldRepository) Create(headerID int64, models []VatFoldModel VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, headerID, model.Ilk, model.Urn, model.Rate, model.TransactionIndex, model.Raw, ) + if err != nil { + tx.Rollback() + return err + } _, err = tx.Exec( `INSERT INTO public.checked_headers (header_id, vat_fold_checked) VALUES($1, $2) From 89ec0a1cab33e066a242be65347516f460b998e8 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 14:06:00 +0300 Subject: [PATCH 29/30] documentation: document config file for transformers --- pkg/transformers/DOCUMENTATION.md | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index b9298baa..ed3e8a4c 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -35,6 +35,7 @@ The transformer process for each of these different log types is the same, excep 1. Write converter + converter tests 1. Write repository + repository tests 1. Create converter + repository mocks +1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go` 1. Create transformer + transformer tests 1. Wire up transformer in [`transformers.go`](./transformers.go) 1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go) From 0bb37370279aec90753f4dba6978751dc307ca49 Mon Sep 17 00:00:00 2001 From: David Terry Date: Wed, 10 Oct 2018 18:13:10 +0300 Subject: [PATCH 30/30] vat.fold: pull common test setup into BeforeEach blocks --- pkg/transformers/vat_fold/repository_test.go | 86 ++++++++------------ 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index 38b28397..16e3c691 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -30,23 +30,26 @@ import ( var _ = Describe("Vat.fold repository", func() { + var db *postgres.DB + var headerID int64 + var repository vat_fold.VatFoldRepository + var headerRepository repositories.HeaderRepository + var err error + + BeforeEach(func() { + node := test_config.NewTestNode() + db = test_config.NewTestDB(node) + test_config.CleanTestDB(db) + repository = vat_fold.NewVatFoldRepository(db) + headerRepository = repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + }) + Describe("Create", func() { - var db *postgres.DB - var headerID int64 - var vatFoldRepository vat_fold.VatFoldRepository - BeforeEach(func() { - db = test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - - headerRepository := repositories.NewHeaderRepository(db) - id, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - headerID = id - - vatFoldRepository = vat_fold.NewVatFoldRepository(db) - err = vatFoldRepository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) + err = repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -63,7 +66,7 @@ var _ = Describe("Vat.fold repository", func() { }) It("does not duplicate vat events", func() { - err := vatFoldRepository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) + err := repository.Create(headerID, []vat_fold.VatFoldModel{test_data.VatFoldModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -81,29 +84,10 @@ var _ = Describe("Vat.fold repository", func() { }) Describe("MarkHeaderChecked", func() { - var db *postgres.DB - var headerID int64 - var repository vat_fold.VatFoldRepository - type CheckedHeaderResult struct { VatFoldChecked bool `db:"vat_fold_checked"` } - BeforeEach(func() { - node := test_config.NewTestNode() - - db = test_config.NewTestDB(node) - test_config.CleanTestDB(db) - - headerRepository := repositories.NewHeaderRepository(db) - id, err := headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - headerID = id - - repository = vat_fold.NewVatFoldRepository(db) - Expect(err).NotTo(HaveOccurred()) - }) - It("creates a new checked header record", func() { err := repository.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) @@ -135,24 +119,22 @@ var _ = Describe("Vat.fold repository", func() { Describe("MissingHeaders", func() { It("returns headers that haven't been checked", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) startingBlockNumber := int64(1) vatGrabBlockNumber := int64(2) endingBlockNumber := int64(3) blockNumbers := []int64{startingBlockNumber, vatGrabBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 for _, n := range blockNumbers { headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) headerIDs = append(headerIDs, headerID) Expect(err).NotTo(HaveOccurred()) } - vatFoldRepository := vat_fold.NewVatFoldRepository(db) - err := vatFoldRepository.MarkHeaderChecked(headerIDs[1]) + + err := repository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := vatFoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(2)) @@ -161,24 +143,22 @@ var _ = Describe("Vat.fold repository", func() { }) It("only treats headers as checked if vat fold logs have been checked", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) startingBlockNumber := int64(1) vatGrabdBlockNumber := int64(2) endingBlockNumber := int64(3) blockNumbers := []int64{startingBlockNumber, vatGrabdBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 for _, n := range blockNumbers { headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) headerIDs = append(headerIDs, headerID) Expect(err).NotTo(HaveOccurred()) } - vatFoldRepository := vat_fold.NewVatFoldRepository(db) + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := vatFoldRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(3)) @@ -188,30 +168,32 @@ var _ = Describe("Vat.fold repository", func() { }) It("only returns headers associated with the current node", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) blockNumbers := []int64{1, 2, 3} headerRepository := repositories.NewHeaderRepository(db) + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 for _, n := range blockNumbers { headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) Expect(err).NotTo(HaveOccurred()) headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) Expect(err).NotTo(HaveOccurred()) } - vatFoldRepository := vat_fold.NewVatFoldRepository(db) - vatFoldRepositoryTwo := vat_fold.NewVatFoldRepository(dbTwo) - err := vatFoldRepository.MarkHeaderChecked(headerIDs[0]) + + repositoryTwo := vat_fold.NewVatFoldRepository(dbTwo) + + err := repository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := vatFoldRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeOneMissingHeaders, err := repository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) - nodeTwoMissingHeaders, err := vatFoldRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) })